{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第三章：加工原料文本\n",
    "解决：\n",
    "1. 编写程序访问本地和网络文件，获得语料\n",
    "2. 把文档分割成单独的词和标点符号，进行预料分析\n",
    "3. 格式化输出，将结果保存到文件"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1 从网络和硬盘访问文本"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1.1 电子书"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk, re, pprint\n",
    "from nltk import word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'str'>\n",
      "1176965\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'The Project Gutenberg EBook of Crime and Punishment, by Fyodor Dostoevsky\\r'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取网络文件\n",
    "from urllib import request\n",
    "url = \"http://www.gutenberg.org/files/2554/2554-0.txt\"#原地址有变化，需根据网页调整\n",
    "response = request.urlopen(url)\n",
    "raw = response.read().decode('utf8')\n",
    "print(type(raw))\n",
    "print(len(raw))\n",
    "raw[1:75]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "257726\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['The',\n",
       " 'Project',\n",
       " 'Gutenberg',\n",
       " 'EBook',\n",
       " 'of',\n",
       " 'Crime',\n",
       " 'and',\n",
       " 'Punishment',\n",
       " ',',\n",
       " 'by']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#使用NTLK分词\n",
    "tokens = word_tokenize(raw[1:])#切片操作去除第一个字符，分词产生一个词汇和标点符号的链表\n",
    "print(type(tokens))\n",
    "print(len(tokens))\n",
    "tokens[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'nltk.text.Text'>\n",
      "['CHAPTER', 'I', 'On', 'an', 'exceptionally', 'hot', 'evening', 'early', 'in', 'July', 'a', 'young', 'man', 'came', 'out', 'of', 'the', 'garret', 'in', 'which', 'he', 'lodged', 'in', 'S.', 'Place', 'and', 'walked', 'slowly', ',', 'as', 'though', 'in', 'hesitation', ',', 'towards', 'K.', 'bridge', '.']\n",
      "Katerina Ivanovna; Pyotr Petrovitch; Pulcheria Alexandrovna; Avdotya\n",
      "Romanovna; Rodion Romanovitch; Marfa Petrovna; Sofya Semyonovna; old\n",
      "woman; Project Gutenberg-tm; Porfiry Petrovitch; Amalia Ivanovna;\n",
      "great deal; young man; Nikodim Fomitch; Ilya Petrovitch; Project\n",
      "Gutenberg; Andrey Semyonovitch; Hay Market; Dmitri Prokofitch; Good\n",
      "heavens\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "#链表操作\n",
    "text = nltk.Text(tokens)\n",
    "print(type(text))\n",
    "print(text[1021:1059])\n",
    "print(text.collocations())#经常一起出现的词序列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5336\n",
      "-1\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "#正向查找find（）及反向查找rfind（）\n",
    "print(raw.find(\"PART I\"))\n",
    "print(raw.rfind(\"End of Project Gutenberg's Crime\"))\n",
    "raw = raw[5336:1157741]\n",
    "print(raw.find(\"PART I\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1.2 处理 HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<!doctype html public \"-//W3C//DTD HTML 4.0 Transitional//EN'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#访问网址获取HTML的全部内容\n",
    "url = \"http://news.bbc.co.uk/2/hi/health/2284783.stm\"\n",
    "html = request.urlopen(url).read().decode('utf8')\n",
    "html[:60]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['BBC',\n",
       " 'NEWS',\n",
       " '|',\n",
       " 'Health',\n",
       " '|',\n",
       " 'Blondes',\n",
       " \"'to\",\n",
       " 'die',\n",
       " 'out',\n",
       " 'in',\n",
       " '200',\n",
       " \"years'\",\n",
       " 'NEWS',\n",
       " 'SPORT',\n",
       " 'WEATHER',\n",
       " 'WORLD',\n",
       " 'SERVICE',\n",
       " 'A-Z',\n",
       " 'INDEX',\n",
       " 'SEARCH',\n",
       " 'You',\n",
       " 'are',\n",
       " 'in',\n",
       " ':',\n",
       " 'Health',\n",
       " 'News',\n",
       " 'Front',\n",
       " 'Page',\n",
       " 'Africa',\n",
       " 'Americas',\n",
       " 'Asia-Pacific',\n",
       " 'Europe',\n",
       " 'Middle',\n",
       " 'East',\n",
       " 'South',\n",
       " 'Asia',\n",
       " 'UK',\n",
       " 'Business',\n",
       " 'Entertainment',\n",
       " 'Science/Nature',\n",
       " 'Technology',\n",
       " 'Health',\n",
       " 'Medical',\n",
       " 'notes',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '-',\n",
       " 'Talking',\n",
       " 'Point',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '-',\n",
       " 'Country',\n",
       " 'Profiles',\n",
       " 'In',\n",
       " 'Depth',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '-',\n",
       " 'Programmes',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '-',\n",
       " 'SERVICES',\n",
       " 'Daily',\n",
       " 'E-mail',\n",
       " 'News',\n",
       " 'Ticker',\n",
       " 'Mobile/PDAs',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '-',\n",
       " 'Text',\n",
       " 'Only',\n",
       " 'Feedback',\n",
       " 'Help',\n",
       " 'EDITIONS',\n",
       " 'Change',\n",
       " 'to',\n",
       " 'UK',\n",
       " 'Friday',\n",
       " ',',\n",
       " '27',\n",
       " 'September',\n",
       " ',',\n",
       " '2002',\n",
       " ',',\n",
       " '11:51',\n",
       " 'GMT',\n",
       " '12:51',\n",
       " 'UK',\n",
       " 'Blondes',\n",
       " \"'to\",\n",
       " 'die',\n",
       " 'out',\n",
       " 'in',\n",
       " '200',\n",
       " \"years'\",\n",
       " 'Scientists',\n",
       " 'believe',\n",
       " 'the',\n",
       " 'last',\n",
       " 'blondes',\n",
       " 'will',\n",
       " 'be',\n",
       " 'in',\n",
       " 'Finland',\n",
       " 'The',\n",
       " 'last',\n",
       " 'natural',\n",
       " 'blondes',\n",
       " 'will',\n",
       " 'die',\n",
       " 'out',\n",
       " 'within',\n",
       " '200',\n",
       " 'years',\n",
       " ',',\n",
       " 'scientists',\n",
       " 'believe',\n",
       " '.',\n",
       " 'A',\n",
       " 'study',\n",
       " 'by',\n",
       " 'experts',\n",
       " 'in',\n",
       " 'Germany',\n",
       " 'suggests',\n",
       " 'people',\n",
       " 'with',\n",
       " 'blonde',\n",
       " 'hair',\n",
       " 'are',\n",
       " 'an',\n",
       " 'endangered',\n",
       " 'species',\n",
       " 'and',\n",
       " 'will',\n",
       " 'become',\n",
       " 'extinct',\n",
       " 'by',\n",
       " '2202',\n",
       " '.',\n",
       " 'Researchers',\n",
       " 'predict',\n",
       " 'the',\n",
       " 'last',\n",
       " 'truly',\n",
       " 'natural',\n",
       " 'blonde',\n",
       " 'will',\n",
       " 'be',\n",
       " 'born',\n",
       " 'in',\n",
       " 'Finland',\n",
       " '-',\n",
       " 'the',\n",
       " 'country',\n",
       " 'with',\n",
       " 'the',\n",
       " 'highest',\n",
       " 'proportion',\n",
       " 'of',\n",
       " 'blondes',\n",
       " '.',\n",
       " 'The',\n",
       " 'frequency',\n",
       " 'of',\n",
       " 'blondes',\n",
       " 'may',\n",
       " 'drop',\n",
       " 'but',\n",
       " 'they',\n",
       " 'wo',\n",
       " \"n't\",\n",
       " 'disappear',\n",
       " 'Prof',\n",
       " 'Jonathan',\n",
       " 'Rees',\n",
       " ',',\n",
       " 'University',\n",
       " 'of',\n",
       " 'Edinburgh',\n",
       " 'But',\n",
       " 'they',\n",
       " 'say',\n",
       " 'too',\n",
       " 'few',\n",
       " 'people',\n",
       " 'now',\n",
       " 'carry',\n",
       " 'the',\n",
       " 'gene',\n",
       " 'for',\n",
       " 'blondes',\n",
       " 'to',\n",
       " 'last',\n",
       " 'beyond',\n",
       " 'the',\n",
       " 'next',\n",
       " 'two',\n",
       " 'centuries',\n",
       " '.',\n",
       " 'The',\n",
       " 'problem',\n",
       " 'is',\n",
       " 'that',\n",
       " 'blonde',\n",
       " 'hair',\n",
       " 'is',\n",
       " 'caused',\n",
       " 'by',\n",
       " 'a',\n",
       " 'recessive',\n",
       " 'gene',\n",
       " '.',\n",
       " 'In',\n",
       " 'order',\n",
       " 'for',\n",
       " 'a',\n",
       " 'child',\n",
       " 'to',\n",
       " 'have',\n",
       " 'blonde',\n",
       " 'hair',\n",
       " ',',\n",
       " 'it',\n",
       " 'must',\n",
       " 'have',\n",
       " 'the',\n",
       " 'gene',\n",
       " 'on',\n",
       " 'both',\n",
       " 'sides',\n",
       " 'of',\n",
       " 'the',\n",
       " 'family',\n",
       " 'in',\n",
       " 'the',\n",
       " 'grandparents',\n",
       " \"'\",\n",
       " 'generation',\n",
       " '.',\n",
       " 'Dyed',\n",
       " 'rivals',\n",
       " 'The',\n",
       " 'researchers',\n",
       " 'also',\n",
       " 'believe',\n",
       " 'that',\n",
       " 'so-called',\n",
       " 'bottle',\n",
       " 'blondes',\n",
       " 'may',\n",
       " 'be',\n",
       " 'to',\n",
       " 'blame',\n",
       " 'for',\n",
       " 'the',\n",
       " 'demise',\n",
       " 'of',\n",
       " 'their',\n",
       " 'natural',\n",
       " 'rivals',\n",
       " '.',\n",
       " 'They',\n",
       " 'suggest',\n",
       " 'that',\n",
       " 'dyed-blondes',\n",
       " 'are',\n",
       " 'more',\n",
       " 'attractive',\n",
       " 'to',\n",
       " 'men',\n",
       " 'who',\n",
       " 'choose',\n",
       " 'them',\n",
       " 'as',\n",
       " 'partners',\n",
       " 'over',\n",
       " 'true',\n",
       " 'blondes',\n",
       " '.',\n",
       " 'Bottle-blondes',\n",
       " 'like',\n",
       " 'Ann',\n",
       " 'Widdecombe',\n",
       " 'may',\n",
       " 'be',\n",
       " 'to',\n",
       " 'blame',\n",
       " 'But',\n",
       " 'Jonathan',\n",
       " 'Rees',\n",
       " ',',\n",
       " 'professor',\n",
       " 'of',\n",
       " 'dermatology',\n",
       " 'at',\n",
       " 'the',\n",
       " 'University',\n",
       " 'of',\n",
       " 'Edinburgh',\n",
       " 'said',\n",
       " 'it',\n",
       " 'was',\n",
       " 'unlikely',\n",
       " 'blondes',\n",
       " 'would',\n",
       " 'die',\n",
       " 'out',\n",
       " 'completely',\n",
       " '.',\n",
       " '``',\n",
       " 'Genes',\n",
       " 'do',\n",
       " \"n't\",\n",
       " 'die',\n",
       " 'out',\n",
       " 'unless',\n",
       " 'there',\n",
       " 'is',\n",
       " 'a',\n",
       " 'disadvantage',\n",
       " 'of',\n",
       " 'having',\n",
       " 'that',\n",
       " 'gene',\n",
       " 'or',\n",
       " 'by',\n",
       " 'chance',\n",
       " '.',\n",
       " 'They',\n",
       " 'do',\n",
       " \"n't\",\n",
       " 'disappear',\n",
       " ',',\n",
       " \"''\",\n",
       " 'he',\n",
       " 'told',\n",
       " 'BBC',\n",
       " 'News',\n",
       " 'Online',\n",
       " '.',\n",
       " '``',\n",
       " 'The',\n",
       " 'only',\n",
       " 'reason',\n",
       " 'blondes',\n",
       " 'would',\n",
       " 'disappear',\n",
       " 'is',\n",
       " 'if',\n",
       " 'having',\n",
       " 'the',\n",
       " 'gene',\n",
       " 'was',\n",
       " 'a',\n",
       " 'disadvantage',\n",
       " 'and',\n",
       " 'I',\n",
       " 'do',\n",
       " 'not',\n",
       " 'think',\n",
       " 'that',\n",
       " 'is',\n",
       " 'the',\n",
       " 'case',\n",
       " '.',\n",
       " '``',\n",
       " 'The',\n",
       " 'frequency',\n",
       " 'of',\n",
       " 'blondes',\n",
       " 'may',\n",
       " 'drop',\n",
       " 'but',\n",
       " 'they',\n",
       " 'wo',\n",
       " \"n't\",\n",
       " 'disappear',\n",
       " '.',\n",
       " \"''\",\n",
       " 'See',\n",
       " 'also',\n",
       " ':',\n",
       " '28',\n",
       " 'Mar',\n",
       " '01',\n",
       " '|',\n",
       " 'Education',\n",
       " 'What',\n",
       " 'is',\n",
       " 'it',\n",
       " 'about',\n",
       " 'blondes',\n",
       " '?',\n",
       " '09',\n",
       " 'Apr',\n",
       " '99',\n",
       " '|',\n",
       " 'Health',\n",
       " 'Platinum',\n",
       " 'blondes',\n",
       " 'are',\n",
       " 'labelled',\n",
       " 'as',\n",
       " 'dumb',\n",
       " '17',\n",
       " 'Apr',\n",
       " '02',\n",
       " '|',\n",
       " 'Health',\n",
       " 'Hair',\n",
       " 'dye',\n",
       " 'cancer',\n",
       " 'alert',\n",
       " 'Internet',\n",
       " 'links',\n",
       " ':',\n",
       " 'University',\n",
       " 'of',\n",
       " 'Edinburgh',\n",
       " 'The',\n",
       " 'BBC',\n",
       " 'is',\n",
       " 'not',\n",
       " 'responsible',\n",
       " 'for',\n",
       " 'the',\n",
       " 'content',\n",
       " 'of',\n",
       " 'external',\n",
       " 'internet',\n",
       " 'sites',\n",
       " 'Top',\n",
       " 'Health',\n",
       " 'stories',\n",
       " 'now',\n",
       " ':',\n",
       " 'Heart',\n",
       " 'risk',\n",
       " 'link',\n",
       " 'to',\n",
       " 'big',\n",
       " 'families',\n",
       " 'Back',\n",
       " 'pain',\n",
       " 'drug',\n",
       " \"'may\",\n",
       " 'aid',\n",
       " \"diabetics'\",\n",
       " 'Congo',\n",
       " 'Ebola',\n",
       " 'outbreak',\n",
       " 'confirmed',\n",
       " 'Vegetables',\n",
       " 'ward',\n",
       " 'off',\n",
       " \"Alzheimer's\",\n",
       " 'Polio',\n",
       " 'campaign',\n",
       " 'launched',\n",
       " 'in',\n",
       " 'Iraq',\n",
       " 'Gene',\n",
       " 'defect',\n",
       " 'explains',\n",
       " 'high',\n",
       " 'blood',\n",
       " 'pressure',\n",
       " 'Botox',\n",
       " \"'may\",\n",
       " 'cause',\n",
       " 'new',\n",
       " \"wrinkles'\",\n",
       " 'Alien',\n",
       " \"'abductees\",\n",
       " \"'\",\n",
       " 'show',\n",
       " 'real',\n",
       " 'symptoms',\n",
       " 'Links',\n",
       " 'to',\n",
       " 'more',\n",
       " 'Health',\n",
       " 'stories',\n",
       " 'are',\n",
       " 'at',\n",
       " 'the',\n",
       " 'foot',\n",
       " 'of',\n",
       " 'the',\n",
       " 'page',\n",
       " '.',\n",
       " 'E-mail',\n",
       " 'this',\n",
       " 'story',\n",
       " 'to',\n",
       " 'a',\n",
       " 'friend',\n",
       " 'Links',\n",
       " 'to',\n",
       " 'more',\n",
       " 'Health',\n",
       " 'stories',\n",
       " 'In',\n",
       " 'This',\n",
       " 'Section',\n",
       " 'Heart',\n",
       " 'risk',\n",
       " 'link',\n",
       " 'to',\n",
       " 'big',\n",
       " 'families',\n",
       " 'Back',\n",
       " 'pain',\n",
       " 'drug',\n",
       " \"'may\",\n",
       " 'aid',\n",
       " \"diabetics'\",\n",
       " 'Congo',\n",
       " 'Ebola',\n",
       " 'outbreak',\n",
       " 'confirmed',\n",
       " 'Vegetables',\n",
       " 'ward',\n",
       " 'off',\n",
       " \"Alzheimer's\",\n",
       " 'Polio',\n",
       " 'campaign',\n",
       " 'launched',\n",
       " 'in',\n",
       " 'Iraq',\n",
       " 'Gene',\n",
       " 'defect',\n",
       " 'explains',\n",
       " 'high',\n",
       " 'blood',\n",
       " 'pressure',\n",
       " 'Botox',\n",
       " \"'may\",\n",
       " 'cause',\n",
       " 'new',\n",
       " \"wrinkles'\",\n",
       " 'Alien',\n",
       " \"'abductees\",\n",
       " \"'\",\n",
       " 'show',\n",
       " 'real',\n",
       " 'symptoms',\n",
       " 'How',\n",
       " 'sperm',\n",
       " 'wriggle',\n",
       " 'Bollywood',\n",
       " 'told',\n",
       " 'to',\n",
       " 'stub',\n",
       " 'it',\n",
       " 'out',\n",
       " 'Fears',\n",
       " 'over',\n",
       " 'tuna',\n",
       " 'health',\n",
       " 'risk',\n",
       " 'to',\n",
       " 'babies',\n",
       " 'Public',\n",
       " 'can',\n",
       " 'be',\n",
       " 'taught',\n",
       " 'to',\n",
       " 'spot',\n",
       " 'strokes',\n",
       " '^^',\n",
       " 'Back',\n",
       " 'to',\n",
       " 'top',\n",
       " 'News',\n",
       " 'Front',\n",
       " 'Page',\n",
       " '|',\n",
       " 'Africa',\n",
       " '|',\n",
       " 'Americas',\n",
       " '|',\n",
       " 'Asia-Pacific',\n",
       " '|',\n",
       " 'Europe',\n",
       " '|',\n",
       " 'Middle',\n",
       " 'East',\n",
       " '|',\n",
       " 'South',\n",
       " 'Asia',\n",
       " '|',\n",
       " 'UK',\n",
       " '|',\n",
       " 'Business',\n",
       " '|',\n",
       " 'Entertainment',\n",
       " '|',\n",
       " 'Science/Nature',\n",
       " '|',\n",
       " 'Technology',\n",
       " '|',\n",
       " 'Health',\n",
       " '|',\n",
       " 'Talking',\n",
       " 'Point',\n",
       " '|',\n",
       " 'Country',\n",
       " 'Profiles',\n",
       " '|',\n",
       " 'In',\n",
       " 'Depth',\n",
       " '|',\n",
       " 'Programmes',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " 'To',\n",
       " 'BBC',\n",
       " 'Sport',\n",
       " '>',\n",
       " '>',\n",
       " '|',\n",
       " 'To',\n",
       " 'BBC',\n",
       " 'Weather',\n",
       " '>',\n",
       " '>',\n",
       " '|',\n",
       " 'To',\n",
       " 'BBC',\n",
       " 'World',\n",
       " 'Service',\n",
       " '>',\n",
       " '>',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '©',\n",
       " 'MMIII',\n",
       " '|',\n",
       " 'News',\n",
       " 'Sources',\n",
       " '|',\n",
       " 'Privacy',\n",
       " '<',\n",
       " '!',\n",
       " '--',\n",
       " 'var',\n",
       " 'pCid=',\n",
       " \"''\",\n",
       " 'uk_bbc_0',\n",
       " \"''\",\n",
       " ';',\n",
       " 'var',\n",
       " 'w0=1',\n",
       " ';',\n",
       " 'var',\n",
       " 'refR=escape',\n",
       " '(',\n",
       " 'document.referrer',\n",
       " ')',\n",
       " ';',\n",
       " 'if',\n",
       " '(',\n",
       " 'refR.length',\n",
       " '>',\n",
       " '=252',\n",
       " ')',\n",
       " 'refR=refR.substring',\n",
       " '(',\n",
       " '0,252',\n",
       " ')',\n",
       " '+',\n",
       " \"''\",\n",
       " '...',\n",
       " \"''\",\n",
       " ';',\n",
       " '//',\n",
       " '--',\n",
       " '>',\n",
       " '<',\n",
       " '!',\n",
       " '--',\n",
       " 'var',\n",
       " 'w0=0',\n",
       " ';',\n",
       " '//',\n",
       " '--',\n",
       " '>',\n",
       " '<',\n",
       " '!',\n",
       " '--',\n",
       " 'if',\n",
       " '(',\n",
       " 'w0',\n",
       " ')',\n",
       " '{',\n",
       " 'var',\n",
       " 'imgN=',\n",
       " \"'\",\n",
       " '<',\n",
       " 'img',\n",
       " 'src=',\n",
       " \"''\",\n",
       " 'http',\n",
       " ':',\n",
       " '//server-uk.imrworldwide.com/cgi-bin/count',\n",
       " '?',\n",
       " \"ref='+\",\n",
       " 'refR+',\n",
       " \"'\",\n",
       " '&',\n",
       " \"cid='+pCid+\",\n",
       " \"'\",\n",
       " \"''\",\n",
       " 'width=1',\n",
       " 'height=1',\n",
       " '>',\n",
       " \"'\",\n",
       " ';',\n",
       " 'if',\n",
       " '(',\n",
       " 'navigator.userAgent.indexOf',\n",
       " '(',\n",
       " \"'Mac\",\n",
       " \"'\",\n",
       " ')',\n",
       " '!',\n",
       " '=-1',\n",
       " ')',\n",
       " '{',\n",
       " 'document.write',\n",
       " '(',\n",
       " 'imgN',\n",
       " ')',\n",
       " ';',\n",
       " '}',\n",
       " 'else',\n",
       " '{',\n",
       " 'document.write',\n",
       " '(',\n",
       " \"'\",\n",
       " '<',\n",
       " 'applet',\n",
       " 'code=',\n",
       " \"''\",\n",
       " 'Measure.class',\n",
       " \"''\",\n",
       " \"'+\",\n",
       " \"'codebase=\",\n",
       " \"''\",\n",
       " 'http',\n",
       " ':',\n",
       " '//server-uk.imrworldwide.com/',\n",
       " \"''\",\n",
       " \"'+'width=1\",\n",
       " 'height=2',\n",
       " '>',\n",
       " \"'+\",\n",
       " \"'\",\n",
       " '<',\n",
       " 'param',\n",
       " 'name=',\n",
       " \"''\",\n",
       " 'ref',\n",
       " \"''\",\n",
       " 'value=',\n",
       " \"''\",\n",
       " \"'+refR+\",\n",
       " \"'\",\n",
       " \"''\",\n",
       " '>',\n",
       " \"'+\",\n",
       " \"'\",\n",
       " '<',\n",
       " 'param',\n",
       " 'name=',\n",
       " \"''\",\n",
       " 'cid',\n",
       " \"''\",\n",
       " 'value=',\n",
       " \"''\",\n",
       " \"'+pCid+\",\n",
       " \"'\",\n",
       " \"''\",\n",
       " '>',\n",
       " '<',\n",
       " 'textflow',\n",
       " '>',\n",
       " \"'+imgN+\",\n",
       " \"'\",\n",
       " '<',\n",
       " '/textflow',\n",
       " '>',\n",
       " '<',\n",
       " '/applet',\n",
       " '>',\n",
       " \"'\",\n",
       " ')',\n",
       " ';',\n",
       " '}',\n",
       " '}',\n",
       " 'document.write',\n",
       " '(',\n",
       " '``',\n",
       " '<',\n",
       " 'COMMENT',\n",
       " '>',\n",
       " \"''\",\n",
       " ')',\n",
       " ';',\n",
       " '//',\n",
       " '--',\n",
       " '>',\n",
       " 'var',\n",
       " 'si',\n",
       " '=',\n",
       " 'document.location+',\n",
       " \"''\",\n",
       " \"''\",\n",
       " ';',\n",
       " 'var',\n",
       " 'tsi',\n",
       " '=',\n",
       " 'si.replace',\n",
       " '(',\n",
       " '``',\n",
       " '.stm',\n",
       " \"''\",\n",
       " ',',\n",
       " \"''\",\n",
       " \"''\",\n",
       " ')',\n",
       " '.substr',\n",
       " '(',\n",
       " 'si.length-11',\n",
       " ',',\n",
       " 'si.length',\n",
       " ')',\n",
       " ';',\n",
       " 'if',\n",
       " '(',\n",
       " '!',\n",
       " 'tsi.match',\n",
       " '(',\n",
       " '/\\\\d\\\\d\\\\d\\\\d\\\\d\\\\d\\\\d/',\n",
       " ')',\n",
       " ')',\n",
       " '{',\n",
       " 'tsi',\n",
       " '=',\n",
       " '0',\n",
       " ';',\n",
       " '}',\n",
       " 'document.write',\n",
       " '(',\n",
       " \"'\",\n",
       " '<',\n",
       " 'img',\n",
       " 'src=',\n",
       " \"''\",\n",
       " 'http',\n",
       " ':',\n",
       " '//stats.bbc.co.uk/o.gif',\n",
       " '?',\n",
       " '~RS~s~RS~News~RS~t~RS~HighWeb_Legacy~RS~i~RS~',\n",
       " \"'\",\n",
       " '+',\n",
       " 'tsi',\n",
       " '+',\n",
       " \"'~RS~p~RS~0~RS~u~RS~/2/hi/health/2284783.stm~RS~r~RS~\",\n",
       " '(',\n",
       " 'none',\n",
       " ')',\n",
       " '~RS~a~RS~International~RS~q~RS~~RS~z~RS~07~RS~',\n",
       " \"''\",\n",
       " '>',\n",
       " \"'\",\n",
       " ')',\n",
       " ';']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#提取原始文本\n",
    "from bs4 import BeautifulSoup\n",
    "raw = BeautifulSoup(html,'lxml').get_text()\n",
    "tokens = word_tokenize(raw)\n",
    "tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying 5 of 5 matches:\n",
      "hey say too few people now carry the gene for blondes to last beyond the next \n",
      "blonde hair is caused by a recessive gene . In order for a child to have blond\n",
      " have blonde hair , it must have the gene on both sides of the family in the g\n",
      "ere is a disadvantage of having that gene or by chance . They do n't disappear\n",
      "des would disappear is if having the gene was a disadvantage and I do not thin\n"
     ]
    }
   ],
   "source": [
    "#搜索单词gene在text 中出现的情况,强调上下文\n",
    "tokens = tokens[110:390]\n",
    "text = nltk.Text(tokens)\n",
    "text.concordance('gene')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1.3 处理搜索引擎结果\n",
    "网络可以被看作未经标注的巨大的语料库。  \n",
    "优势：  \n",
    "规模  \n",
    "限制：  \n",
    "搜索范围受到限制  \n",
    "搜索引擎一般只允许搜索单个词或词串及使用通配符。  \n",
    "搜索引擎给出的结果不一致，在不同的时间或区域给出不同结果  \n",
    "搜索引擎返回的结果中的标记可能会不可预料的改变  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1.4 处理 RSS 订阅"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Language Log\n",
      "13\n"
     ]
    }
   ],
   "source": [
    "#Universal Feed Parser第三方Python库,访问博客的内容\n",
    "import feedparser\n",
    "llog = feedparser.parse(\"http://languagelog.ldc.upenn.edu/nll/?feed=atom\")\n",
    "print(llog['feed']['title'])\n",
    "print(len(llog.entries))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Year Hare Affair'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#访问第2个博客内容\n",
    "post = llog.entries[1]\n",
    "post.title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"<p>That's the abbreviated title of a popular webcomic by Lin Chao 林超.\\xa0\""
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#提取博客内容\n",
    "content = post.content[0].value\n",
    "content[:70]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['That',\n",
       " \"'s\",\n",
       " 'the',\n",
       " 'abbreviated',\n",
       " 'title',\n",
       " 'of',\n",
       " 'a',\n",
       " 'popular',\n",
       " 'webcomic',\n",
       " 'by',\n",
       " 'Lin',\n",
       " 'Chao',\n",
       " '林超',\n",
       " '.',\n",
       " 'The',\n",
       " 'full',\n",
       " 'title',\n",
       " 'in',\n",
       " 'Chinese',\n",
       " 'is',\n",
       " 'Nà',\n",
       " 'nián',\n",
       " 'nà',\n",
       " 'tù',\n",
       " 'nàxiē',\n",
       " 'shì',\n",
       " '那年那兔那些事',\n",
       " '(',\n",
       " 'lit.',\n",
       " ',',\n",
       " '``',\n",
       " 'that',\n",
       " 'year',\n",
       " 'that',\n",
       " 'rabbit',\n",
       " 'those',\n",
       " 'affairs',\n",
       " \"''\",\n",
       " ';',\n",
       " 'i.e.',\n",
       " ',',\n",
       " '``',\n",
       " 'The',\n",
       " 'story',\n",
       " 'of',\n",
       " 'that',\n",
       " 'rabbit',\n",
       " 'that',\n",
       " 'happened',\n",
       " 'in',\n",
       " 'that',\n",
       " 'year',\n",
       " \"''\",\n",
       " ')',\n",
       " 'From',\n",
       " 'the',\n",
       " 'beginning',\n",
       " 'of',\n",
       " 'the',\n",
       " 'Wikipedia',\n",
       " 'article',\n",
       " ':',\n",
       " 'The',\n",
       " 'comic',\n",
       " 'uses',\n",
       " 'animals',\n",
       " 'as',\n",
       " 'an',\n",
       " 'allegory',\n",
       " 'for',\n",
       " 'nations',\n",
       " 'and',\n",
       " 'sovereign',\n",
       " 'states',\n",
       " 'to',\n",
       " 'represent',\n",
       " 'political',\n",
       " 'and',\n",
       " 'military',\n",
       " 'events',\n",
       " 'in',\n",
       " 'history',\n",
       " '.',\n",
       " 'The',\n",
       " 'goal',\n",
       " 'of',\n",
       " 'this',\n",
       " 'project',\n",
       " 'was',\n",
       " 'to',\n",
       " 'promote',\n",
       " 'nationalistic',\n",
       " 'pride',\n",
       " 'in',\n",
       " 'young',\n",
       " 'people',\n",
       " ',',\n",
       " 'and',\n",
       " 'focuses',\n",
       " 'on',\n",
       " 'appreciation',\n",
       " 'for',\n",
       " 'China',\n",
       " \"'s\",\n",
       " 'various',\n",
       " 'achievements',\n",
       " 'since',\n",
       " 'the',\n",
       " 'beginning',\n",
       " 'of',\n",
       " 'the',\n",
       " '20th',\n",
       " 'century',\n",
       " '.',\n",
       " 'Here',\n",
       " 'is',\n",
       " 'a',\n",
       " 'list',\n",
       " 'of',\n",
       " 'the',\n",
       " 'countries',\n",
       " 'portrayed',\n",
       " 'in',\n",
       " 'the',\n",
       " 'webcomic',\n",
       " 'and',\n",
       " 'the',\n",
       " 'animals',\n",
       " 'used',\n",
       " 'to',\n",
       " 'represent',\n",
       " 'them',\n",
       " ':',\n",
       " 'Appearance',\n",
       " 'Reasons',\n",
       " 'and',\n",
       " 'source',\n",
       " 'of',\n",
       " 'the',\n",
       " 'appearance',\n",
       " 'China',\n",
       " '(',\n",
       " 'or',\n",
       " 'Communist',\n",
       " 'Party',\n",
       " 'of',\n",
       " 'China',\n",
       " 'Hare',\n",
       " 'Hares',\n",
       " 'are',\n",
       " 'herbivorous',\n",
       " 'animals',\n",
       " 'that',\n",
       " 'are',\n",
       " 'usually',\n",
       " 'considered',\n",
       " 'cute',\n",
       " ',',\n",
       " 'docile',\n",
       " 'and',\n",
       " 'populous',\n",
       " ',',\n",
       " 'but',\n",
       " 'can',\n",
       " 'still',\n",
       " 'inflict',\n",
       " 'nasty',\n",
       " 'bites',\n",
       " 'and',\n",
       " 'kicks',\n",
       " 'when',\n",
       " 'irritated',\n",
       " '.',\n",
       " 'It',\n",
       " 'represents',\n",
       " 'the',\n",
       " 'author',\n",
       " \"'s\",\n",
       " 'view',\n",
       " 'that',\n",
       " 'the',\n",
       " 'People',\n",
       " \"'s\",\n",
       " 'Republic',\n",
       " 'of',\n",
       " 'China',\n",
       " 'is',\n",
       " 'traditionally',\n",
       " 'not',\n",
       " 'so',\n",
       " 'aggressive',\n",
       " 'in',\n",
       " 'foreign',\n",
       " 'policies',\n",
       " ',',\n",
       " 'but',\n",
       " 'can',\n",
       " 'still',\n",
       " 'pack',\n",
       " 'a',\n",
       " 'punch',\n",
       " 'when',\n",
       " 'required',\n",
       " '.',\n",
       " 'Taiwan',\n",
       " '(',\n",
       " 'or',\n",
       " 'Kuomintang',\n",
       " ')',\n",
       " 'Baldhead',\n",
       " 'Based',\n",
       " 'on',\n",
       " 'the',\n",
       " 'hair',\n",
       " 'style',\n",
       " 'of',\n",
       " 'Chiang',\n",
       " 'Kai-shek',\n",
       " '.',\n",
       " 'Besides',\n",
       " ',',\n",
       " 'in',\n",
       " 'Standard',\n",
       " 'Chinese',\n",
       " '``',\n",
       " 'baldhead',\n",
       " \"''\",\n",
       " '(',\n",
       " '秃子',\n",
       " 'tūzi',\n",
       " ')',\n",
       " 'and',\n",
       " '``',\n",
       " 'rabbit',\n",
       " \"''\",\n",
       " '(',\n",
       " '兔子',\n",
       " 'tùzi',\n",
       " ')',\n",
       " 'have',\n",
       " 'similar',\n",
       " 'pronunciations',\n",
       " ',',\n",
       " 'which',\n",
       " 'represent',\n",
       " 'the',\n",
       " 'complexity',\n",
       " 'of',\n",
       " 'Cross-Strait',\n",
       " 'relations',\n",
       " '.',\n",
       " 'Soviet',\n",
       " 'Union',\n",
       " 'Russian',\n",
       " 'Bear',\n",
       " 'Bear',\n",
       " 'is',\n",
       " 'a',\n",
       " 'common',\n",
       " 'national',\n",
       " 'personification',\n",
       " 'for',\n",
       " 'Russia',\n",
       " 'and',\n",
       " 'the',\n",
       " 'USSR',\n",
       " 'starting',\n",
       " 'from',\n",
       " 'the',\n",
       " 'Russian',\n",
       " 'Empire',\n",
       " '.',\n",
       " 'The',\n",
       " 'Soviet',\n",
       " 'bear',\n",
       " 'has',\n",
       " 'a',\n",
       " 'symbol',\n",
       " '—',\n",
       " 'the',\n",
       " 'hammer',\n",
       " 'and',\n",
       " 'sickle',\n",
       " 'of',\n",
       " 'Communism',\n",
       " '—',\n",
       " 'on',\n",
       " 'his',\n",
       " 'stomach',\n",
       " ',',\n",
       " 'which',\n",
       " 'later',\n",
       " 'becomes',\n",
       " 'more',\n",
       " 'like',\n",
       " 'a',\n",
       " 'character',\n",
       " ',',\n",
       " '``',\n",
       " '父',\n",
       " \"''\",\n",
       " ',',\n",
       " 'meaning',\n",
       " '``',\n",
       " 'father',\n",
       " \"''\",\n",
       " 'in',\n",
       " 'Chinese',\n",
       " '.',\n",
       " 'Russia',\n",
       " 'Russian',\n",
       " 'Bear',\n",
       " 'with',\n",
       " 'a',\n",
       " 'single',\n",
       " 'separated',\n",
       " 'hair',\n",
       " 'To',\n",
       " 'differentiate',\n",
       " 'with',\n",
       " 'the',\n",
       " 'Soviet',\n",
       " 'Union',\n",
       " '.',\n",
       " 'This',\n",
       " 'bear',\n",
       " 'has',\n",
       " 'a',\n",
       " 'character',\n",
       " '``',\n",
       " '大',\n",
       " \"''\",\n",
       " 'on',\n",
       " 'his',\n",
       " 'stomach',\n",
       " ',',\n",
       " 'meaning',\n",
       " 'that',\n",
       " 'he',\n",
       " 'is',\n",
       " 'the',\n",
       " '``',\n",
       " 'eldest',\n",
       " 'son',\n",
       " \"''\",\n",
       " 'of',\n",
       " 'the',\n",
       " 'USSR',\n",
       " '.',\n",
       " 'The',\n",
       " 'hair',\n",
       " 'stands',\n",
       " 'for',\n",
       " 'the',\n",
       " 'common',\n",
       " 'Northern',\n",
       " 'Chinese',\n",
       " 'nickname',\n",
       " 'for',\n",
       " 'Russians',\n",
       " '``',\n",
       " '毛子',\n",
       " \"''\",\n",
       " '(',\n",
       " '``',\n",
       " 'hairy',\n",
       " 'ones',\n",
       " \"''\",\n",
       " ',',\n",
       " 'referring',\n",
       " 'to',\n",
       " 'the',\n",
       " 'comparatively',\n",
       " 'higher',\n",
       " 'body',\n",
       " 'hair',\n",
       " 'count',\n",
       " 'in',\n",
       " 'Caucasians',\n",
       " ')',\n",
       " '.',\n",
       " 'United',\n",
       " 'States',\n",
       " 'Bald',\n",
       " 'eagle',\n",
       " 'Bald',\n",
       " 'eagle',\n",
       " 'is',\n",
       " 'the',\n",
       " 'national',\n",
       " 'bird',\n",
       " 'of',\n",
       " 'the',\n",
       " 'United',\n",
       " 'States',\n",
       " 'of',\n",
       " 'America',\n",
       " '.',\n",
       " 'Japan',\n",
       " 'Crane',\n",
       " '/',\n",
       " 'Chicken',\n",
       " 'The',\n",
       " 'crane',\n",
       " 'is',\n",
       " 'an',\n",
       " 'important',\n",
       " 'part',\n",
       " 'of',\n",
       " 'the',\n",
       " 'Japanese',\n",
       " 'culture',\n",
       " '.',\n",
       " 'The',\n",
       " 'chicken',\n",
       " 'is',\n",
       " 'used',\n",
       " 'as',\n",
       " 'an',\n",
       " 'insult',\n",
       " 'based',\n",
       " 'on',\n",
       " 'the',\n",
       " 'similar',\n",
       " 'appearance',\n",
       " 'of',\n",
       " 'crane',\n",
       " 'and',\n",
       " 'chicken',\n",
       " 'and',\n",
       " 'the',\n",
       " 'traditional',\n",
       " 'Chinese',\n",
       " 'slang',\n",
       " 'term',\n",
       " 'xiao',\n",
       " 'riben',\n",
       " '.',\n",
       " 'South',\n",
       " 'Korea',\n",
       " 'Gaoli',\n",
       " 'bangzi',\n",
       " 'with',\n",
       " 'helmet',\n",
       " 'Bangzi',\n",
       " 'is',\n",
       " 'an',\n",
       " 'ethnic',\n",
       " 'slur',\n",
       " 'used',\n",
       " 'in',\n",
       " 'Northeast',\n",
       " 'China',\n",
       " 'as',\n",
       " 'a',\n",
       " 'reference',\n",
       " 'of',\n",
       " 'Koreans',\n",
       " '.',\n",
       " 'To',\n",
       " 'differentiate',\n",
       " 'with',\n",
       " 'North',\n",
       " 'Korea',\n",
       " ',',\n",
       " 'a',\n",
       " 'US-style',\n",
       " 'combat',\n",
       " 'helmet',\n",
       " 'is',\n",
       " 'added',\n",
       " 'to',\n",
       " 'the',\n",
       " 'appearance',\n",
       " 'of',\n",
       " 'South',\n",
       " 'Korea',\n",
       " '.',\n",
       " 'North',\n",
       " 'Korea',\n",
       " 'Gaoli',\n",
       " 'bangzi',\n",
       " 'with',\n",
       " 'red',\n",
       " 'star',\n",
       " 'hat',\n",
       " 'Bangzi',\n",
       " 'is',\n",
       " 'an',\n",
       " 'ethnic',\n",
       " 'slur',\n",
       " 'used',\n",
       " 'in',\n",
       " 'Northeast',\n",
       " 'China',\n",
       " 'as',\n",
       " 'a',\n",
       " 'reference',\n",
       " 'of',\n",
       " 'Koreans',\n",
       " '.',\n",
       " 'To',\n",
       " 'differentiate',\n",
       " 'with',\n",
       " 'South',\n",
       " 'Korea',\n",
       " ',',\n",
       " 'a',\n",
       " 'Communist-style',\n",
       " 'red',\n",
       " 'star',\n",
       " 'ski',\n",
       " 'cap',\n",
       " 'is',\n",
       " 'added',\n",
       " 'to',\n",
       " 'the',\n",
       " 'appearance',\n",
       " 'of',\n",
       " 'North',\n",
       " 'Korea',\n",
       " '.',\n",
       " 'Multiple',\n",
       " 'countries',\n",
       " 'in',\n",
       " 'Southeast',\n",
       " 'Asia',\n",
       " 'Vietnam',\n",
       " ',',\n",
       " 'Indonesia',\n",
       " ',',\n",
       " 'Philippines',\n",
       " 'Monkey',\n",
       " 'Monkey',\n",
       " 'is',\n",
       " 'commonly',\n",
       " 'seen',\n",
       " 'in',\n",
       " 'the',\n",
       " 'tropical',\n",
       " 'jungles',\n",
       " 'of',\n",
       " 'Southeast',\n",
       " 'Asia',\n",
       " '.',\n",
       " 'It',\n",
       " 'is',\n",
       " 'also',\n",
       " 'used',\n",
       " 'as',\n",
       " 'an',\n",
       " 'ethnic',\n",
       " 'slur',\n",
       " 'in',\n",
       " 'China',\n",
       " 'for',\n",
       " 'areas',\n",
       " 'without',\n",
       " 'modern',\n",
       " 'development',\n",
       " '.',\n",
       " 'Pakistan',\n",
       " 'Markhor',\n",
       " 'Markhor',\n",
       " 'is',\n",
       " 'commonly',\n",
       " 'seen',\n",
       " 'in',\n",
       " 'Pakistan',\n",
       " 'and',\n",
       " 'the',\n",
       " 'national',\n",
       " 'animal',\n",
       " 'of',\n",
       " 'this',\n",
       " 'country',\n",
       " '.',\n",
       " 'In',\n",
       " 'the',\n",
       " 'animation',\n",
       " 'the',\n",
       " 'Markhor',\n",
       " 'is',\n",
       " 'called',\n",
       " 'Ba',\n",
       " 'Ba',\n",
       " 'Yang',\n",
       " '(',\n",
       " '``',\n",
       " 'Paki',\n",
       " 'Goat',\n",
       " \"''\",\n",
       " ')',\n",
       " 'and',\n",
       " 'sometimes',\n",
       " 'nicknamed',\n",
       " '``',\n",
       " 'Little',\n",
       " 'Ba',\n",
       " \"''\",\n",
       " 'by',\n",
       " 'the',\n",
       " 'Hare',\n",
       " '.',\n",
       " 'India',\n",
       " 'White',\n",
       " 'elephant',\n",
       " 'White',\n",
       " 'elephant',\n",
       " 'is',\n",
       " 'commonly',\n",
       " 'seen',\n",
       " 'in',\n",
       " 'India',\n",
       " 'as',\n",
       " 'an',\n",
       " 'important',\n",
       " 'part',\n",
       " 'of',\n",
       " 'Hindu',\n",
       " 'mythology',\n",
       " '.',\n",
       " 'In',\n",
       " 'the',\n",
       " 'animation',\n",
       " 'the',\n",
       " 'author',\n",
       " 'chooses',\n",
       " 'it',\n",
       " 'rather',\n",
       " 'than',\n",
       " 'the',\n",
       " 'bull',\n",
       " 'which',\n",
       " 'is',\n",
       " 'sacred',\n",
       " 'in',\n",
       " 'Hindu',\n",
       " 'to',\n",
       " 'represent',\n",
       " 'India',\n",
       " 'because',\n",
       " 'bull',\n",
       " 'has',\n",
       " 'already',\n",
       " 'been',\n",
       " 'used',\n",
       " 'to',\n",
       " 'represent',\n",
       " 'the',\n",
       " 'UK',\n",
       " '.',\n",
       " 'United',\n",
       " 'Kingdom',\n",
       " 'Bull',\n",
       " \"''\",\n",
       " 'John',\n",
       " 'Bull',\n",
       " \"''\",\n",
       " 'is',\n",
       " 'a',\n",
       " 'national',\n",
       " 'personification',\n",
       " 'of',\n",
       " 'England',\n",
       " 'and',\n",
       " 'the',\n",
       " 'United',\n",
       " 'Kingdom',\n",
       " 'in',\n",
       " 'general',\n",
       " '.',\n",
       " 'France',\n",
       " 'Gallic',\n",
       " 'rooster',\n",
       " 'Gallic',\n",
       " 'rooster',\n",
       " 'is',\n",
       " 'an',\n",
       " 'unofficial',\n",
       " 'national',\n",
       " 'symbol',\n",
       " 'of',\n",
       " 'France',\n",
       " '.',\n",
       " 'Germany',\n",
       " 'Tiger/Cat',\n",
       " 'Tiger',\n",
       " 'II',\n",
       " 'and',\n",
       " 'other',\n",
       " 'Tiger',\n",
       " 'series',\n",
       " 'of',\n",
       " 'tanks',\n",
       " 'are',\n",
       " 'the',\n",
       " 'most',\n",
       " 'famous',\n",
       " 'German',\n",
       " 'heavy',\n",
       " 'tank',\n",
       " '.',\n",
       " 'So',\n",
       " 'in',\n",
       " 'the',\n",
       " 'assumption',\n",
       " 'of',\n",
       " 'the',\n",
       " 'comic',\n",
       " 'this',\n",
       " 'animal',\n",
       " 'called',\n",
       " 'Hans',\n",
       " 'was',\n",
       " 'at',\n",
       " 'first',\n",
       " 'a',\n",
       " 'tiger',\n",
       " '.',\n",
       " 'But',\n",
       " 'after',\n",
       " 'World',\n",
       " 'War',\n",
       " 'II',\n",
       " ',',\n",
       " 'Germany',\n",
       " 'has',\n",
       " 'been',\n",
       " 'restricted',\n",
       " 'to',\n",
       " 'use',\n",
       " 'military',\n",
       " 'force',\n",
       " ',',\n",
       " 'and',\n",
       " 'thus',\n",
       " 'after',\n",
       " 'the',\n",
       " 'war',\n",
       " 'both',\n",
       " 'East',\n",
       " 'and',\n",
       " 'West',\n",
       " 'Germany',\n",
       " 'become',\n",
       " 'cats',\n",
       " '—',\n",
       " 'a',\n",
       " '``',\n",
       " 'downsized',\n",
       " \"''\",\n",
       " 'tiger',\n",
       " '.',\n",
       " 'Multiple',\n",
       " 'countries',\n",
       " 'in',\n",
       " 'Africa',\n",
       " 'Libyan',\n",
       " 'Arab',\n",
       " 'Jamahiriya',\n",
       " 'Tanzania',\n",
       " 'Sudan',\n",
       " 'South',\n",
       " 'Sudan',\n",
       " 'Eritrea',\n",
       " 'Hippopotamus',\n",
       " 'Hippo',\n",
       " 'is',\n",
       " 'a',\n",
       " 'large',\n",
       " ',',\n",
       " 'mostly',\n",
       " 'herbivorous',\n",
       " 'mammal',\n",
       " 'in',\n",
       " 'Sub-Saharan',\n",
       " 'Africa',\n",
       " '.',\n",
       " 'Both',\n",
       " 'in',\n",
       " 'the',\n",
       " 'comic',\n",
       " 'and',\n",
       " 'the',\n",
       " 'animation',\n",
       " ',',\n",
       " 'nearly',\n",
       " 'all',\n",
       " 'mentioned',\n",
       " 'African',\n",
       " 'countries',\n",
       " 'are',\n",
       " 'represented',\n",
       " 'by',\n",
       " 'hippopotamus',\n",
       " ',',\n",
       " 'except',\n",
       " 'for',\n",
       " 'Uganda',\n",
       " 'under',\n",
       " 'the',\n",
       " 'Idi',\n",
       " 'Amin',\n",
       " 'regime',\n",
       " '.',\n",
       " 'Also',\n",
       " ',',\n",
       " 'Colonel',\n",
       " 'Ka',\n",
       " 'the',\n",
       " 'Hippo',\n",
       " 'sometimes',\n",
       " 'symbolizes',\n",
       " 'Muammar',\n",
       " 'Gaddafi',\n",
       " 'himself',\n",
       " 'other',\n",
       " 'than',\n",
       " 'the',\n",
       " 'country',\n",
       " '.',\n",
       " 'Sudan',\n",
       " 'and',\n",
       " 'South',\n",
       " 'Sudan',\n",
       " 'only',\n",
       " 'appear',\n",
       " 'in',\n",
       " 'the',\n",
       " 'end',\n",
       " 'of',\n",
       " 'Episode',\n",
       " '3',\n",
       " ',',\n",
       " 'Season',\n",
       " '2',\n",
       " 'of',\n",
       " 'the',\n",
       " 'animation',\n",
       " '.',\n",
       " 'Uganda',\n",
       " 'Duck',\n",
       " \"''\",\n",
       " 'Uncle',\n",
       " 'Crazy',\n",
       " 'Duck',\n",
       " \"''\",\n",
       " 'is',\n",
       " ',',\n",
       " 'in',\n",
       " 'fact',\n",
       " ',',\n",
       " 'the',\n",
       " 'nickname',\n",
       " 'of',\n",
       " 'Colonel',\n",
       " 'Muammar',\n",
       " 'Gaddafi',\n",
       " 'among',\n",
       " 'Chinese',\n",
       " 'netizens',\n",
       " '.',\n",
       " 'The',\n",
       " 'appearance',\n",
       " 'of',\n",
       " '``',\n",
       " 'Uncle',\n",
       " 'Crazy',\n",
       " 'Duck',\n",
       " \"''\",\n",
       " 'in',\n",
       " 'the',\n",
       " 'comic',\n",
       " 'is',\n",
       " 'based',\n",
       " 'upon',\n",
       " 'Count',\n",
       " 'Duckula',\n",
       " 'and',\n",
       " 'has',\n",
       " 'screws',\n",
       " 'on',\n",
       " 'his',\n",
       " 'head',\n",
       " ',',\n",
       " 'meaning',\n",
       " 'that',\n",
       " 'his',\n",
       " 'brain',\n",
       " 'is',\n",
       " 'different',\n",
       " 'from',\n",
       " 'other',\n",
       " 'Africans',\n",
       " ';',\n",
       " 'he',\n",
       " 'wears',\n",
       " 'the',\n",
       " 'skin',\n",
       " 'of',\n",
       " 'a',\n",
       " 'hippo',\n",
       " '.',\n",
       " 'In',\n",
       " 'the',\n",
       " 'animation',\n",
       " ',',\n",
       " 'Uncle',\n",
       " 'Crazy',\n",
       " 'Duck',\n",
       " 'even',\n",
       " 'shouts',\n",
       " '``',\n",
       " 'Banana',\n",
       " '!',\n",
       " \"''\",\n",
       " 'as',\n",
       " 'the',\n",
       " 'Minions',\n",
       " 'do',\n",
       " 'during',\n",
       " 'the',\n",
       " 'speech',\n",
       " '.',\n",
       " 'Multiple',\n",
       " 'countries',\n",
       " 'in',\n",
       " 'the',\n",
       " 'Arab',\n",
       " 'world',\n",
       " 'Afghanistan',\n",
       " 'Saudi',\n",
       " 'Arabia',\n",
       " 'Iraq',\n",
       " 'Iran',\n",
       " 'Camels',\n",
       " 'Camel',\n",
       " 'is',\n",
       " 'commonly',\n",
       " 'seen',\n",
       " 'and',\n",
       " 'used',\n",
       " 'as',\n",
       " 'transportation',\n",
       " 'in',\n",
       " 'the',\n",
       " 'desertous',\n",
       " 'Arab',\n",
       " 'world',\n",
       " '.',\n",
       " 'Both',\n",
       " 'in',\n",
       " 'the',\n",
       " 'comic',\n",
       " 'and',\n",
       " 'the',\n",
       " 'animation',\n",
       " ',',\n",
       " 'nearly',\n",
       " 'all',\n",
       " 'mentioned',\n",
       " 'Arabic',\n",
       " 'countries',\n",
       " 'are',\n",
       " 'represented',\n",
       " 'by',\n",
       " 'camels',\n",
       " ',',\n",
       " 'but',\n",
       " 'they',\n",
       " 'have',\n",
       " 'different',\n",
       " 'appearance',\n",
       " '—',\n",
       " 'Afghan',\n",
       " 'camel',\n",
       " 'wears',\n",
       " 'a',\n",
       " 'scarf',\n",
       " 'and',\n",
       " 'is',\n",
       " 'bearded',\n",
       " ';',\n",
       " 'Saudi',\n",
       " 'Arabian',\n",
       " 'camel',\n",
       " 'wears',\n",
       " 'a',\n",
       " 'scarf',\n",
       " 'and',\n",
       " 'lots',\n",
       " 'of',\n",
       " 'diamonds',\n",
       " ';',\n",
       " 'Iraqi',\n",
       " 'camel',\n",
       " 'under',\n",
       " 'the',\n",
       " 'Saddam',\n",
       " ...]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw = BeautifulSoup(content,'lxml').get_text()\n",
    "word_tokenize(raw)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1.5 读取本地文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.android',\n",
       " '.astropy',\n",
       " '.bash_history',\n",
       " '.conda',\n",
       " '.continuum',\n",
       " '.defaults-0.1.0.ini',\n",
       " '.eclipse',\n",
       " '.idlerc',\n",
       " '.ipynb_checkpoints',\n",
       " '.ipython',\n",
       " '.jupyter',\n",
       " '.keras',\n",
       " '.m2',\n",
       " '.matplotlib',\n",
       " '.oracle_jre_usage',\n",
       " '.p2',\n",
       " '.PyCharmCE2017.3',\n",
       " '.spyder-py3',\n",
       " '.ssh',\n",
       " '.theanorc.txt',\n",
       " '.VirtualBox',\n",
       " '1.baseline+script.ipynb',\n",
       " '3D Objects',\n",
       " 'Anaconda3',\n",
       " 'AppData',\n",
       " 'Application Data',\n",
       " 'chg.txt',\n",
       " 'Contacts',\n",
       " 'Cookies',\n",
       " 'data',\n",
       " 'Desktop',\n",
       " 'Documents',\n",
       " 'Downloads',\n",
       " 'dump.raw.txt',\n",
       " 'DUTIRTone',\n",
       " 'Favorites',\n",
       " 'gbdt.csv',\n",
       " 'IntelGraphicsProfiles',\n",
       " 'Links',\n",
       " 'Local Settings',\n",
       " 'logs',\n",
       " 'logs1',\n",
       " 'lr.csv',\n",
       " 'matplotlib.ipynb',\n",
       " 'MiCloud',\n",
       " 'MNIST_data',\n",
       " 'Music',\n",
       " 'My Documents',\n",
       " 'NetHood',\n",
       " 'nlp',\n",
       " 'nlp.ipynb',\n",
       " 'NLP2.ipynb',\n",
       " 'notebooks',\n",
       " 'NTUSER.DAT',\n",
       " 'ntuser.dat.LOG1',\n",
       " 'ntuser.dat.LOG2',\n",
       " 'NTUSER.DAT{ea49ceec-2e07-11e8-81b7-b8ec32391728}.TxR.0.regtrans-ms',\n",
       " 'NTUSER.DAT{ea49ceec-2e07-11e8-81b7-b8ec32391728}.TxR.1.regtrans-ms',\n",
       " 'NTUSER.DAT{ea49ceec-2e07-11e8-81b7-b8ec32391728}.TxR.2.regtrans-ms',\n",
       " 'NTUSER.DAT{ea49ceec-2e07-11e8-81b7-b8ec32391728}.TxR.blf',\n",
       " 'NTUSER.DAT{ea49ceed-2e07-11e8-81b7-b8ec32391728}.TM.blf',\n",
       " 'NTUSER.DAT{ea49ceed-2e07-11e8-81b7-b8ec32391728}.TMContainer00000000000000000001.regtrans-ms',\n",
       " 'NTUSER.DAT{ea49ceed-2e07-11e8-81b7-b8ec32391728}.TMContainer00000000000000000002.regtrans-ms',\n",
       " 'ntuser.ini',\n",
       " 'numpy.ipynb',\n",
       " 'OneDrive',\n",
       " 'output.txt',\n",
       " 'pandas.ipynb',\n",
       " 'Pictures',\n",
       " 'PrintHood',\n",
       " 'r.ipynb',\n",
       " 'r2.ipynb',\n",
       " 'Recent',\n",
       " 'rf1.csv',\n",
       " 'rf2.csv',\n",
       " 'Saved Games',\n",
       " 'search',\n",
       " 'Searches',\n",
       " 'SendTo',\n",
       " 'sklearn.ipynb',\n",
       " 'SmpCup2016-master',\n",
       " 'src',\n",
       " 't.txt',\n",
       " 'temp.csv',\n",
       " 'Templates',\n",
       " 'tensorboard.ipynb',\n",
       " 'tensorflow classification.ipynb',\n",
       " 'tensorflow dropout.ipynb',\n",
       " 'tensorflow.ipynb',\n",
       " 'test.csv',\n",
       " 'train',\n",
       " 'Untitled Folder',\n",
       " 'Untitled.ipynb',\n",
       " 'Untitled1.ipynb',\n",
       " 'Untitled2.ipynb',\n",
       " 'Untitled3.ipynb',\n",
       " 'Untitled4.ipynb',\n",
       " 'valid',\n",
       " 'Videos',\n",
       " 'VirtualBox VMs',\n",
       " 'xgboost',\n",
       " '「开始」菜单']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#列出当前目录\n",
    "os.listdir('.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'汉皇重色思倾国，御宇多年求不得。\\n杨家有女初长成，养在深闺人未识。\\n天生丽质难自弃，一朝选在君王侧。\\n'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#打开并读取文件\n",
    "f = open('chg.txt')\n",
    "f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "汉皇重色思倾国，御宇多年求不得。\n",
      "杨家有女初长成，养在深闺人未识。\n",
      "天生丽质难自弃，一朝选在君王侧。\n"
     ]
    }
   ],
   "source": [
    "#打开chg文件，按行读取，删除空白符\n",
    "f = open('chg.txt', 'r')\n",
    "for line in f:\n",
    "    print(line.strip())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1.6 从PDF、MS Word及其他二进制文件中提取文本\n",
    "PDF及MSword可以借助第三方函数库如pypdf、pywin32进行访问"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1.7 捕获用户输入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter some text: this is a test\n"
     ]
    }
   ],
   "source": [
    "s = input(\"Enter some text: \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You typed 4 words.\n"
     ]
    }
   ],
   "source": [
    "print(\"You typed\", len(word_tokenize(s)), \"words.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2 字符串：最底层的文本处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2.1 字符串的基本操作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Monty Python\n",
      "Monty Python's Flying Circus\n",
      "Monty Python's Flying Circus\n"
     ]
    }
   ],
   "source": [
    "monty = 'Monty Python'\n",
    "circus = \"Monty Python's Flying Circus\"\n",
    "print(monty)\n",
    "print(circus)\n",
    "circus = 'Monty Python\\'s Flying Circus'\n",
    "print(circus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-21-b0616ecb83be>, line 2)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-21-b0616ecb83be>\"\u001b[1;36m, line \u001b[1;32m2\u001b[0m\n\u001b[1;33m    circus = 'Monty Python's Flying Circus'\u001b[0m\n\u001b[1;37m                           ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "#反斜杠的作用\n",
    "circus = 'Monty Python's Flying Circus'\n",
    "print(circus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shall I compare thee to a Summer's day?Thou are more lovely and more temperate:\n",
      "Rough winds do shake the darling buds of May,And Summer's lease hath all too short a date:\n",
      "Shall I compare thee to a Summer's day?\n",
      "Thou are more lovely and more temperate:\n"
     ]
    }
   ],
   "source": [
    "#输出显示换行\n",
    "couplet1 = \"Shall I compare thee to a Summer's day?\"\\\n",
    "            \"Thou are more lovely and more temperate:\" \n",
    "couplet2 = (\"Rough winds do shake the darling buds of May,\"\n",
    "            \"And Summer's lease hath all too short a date:\")\n",
    "couplet3 = \"\"\"Shall I compare thee to a Summer's day?\n",
    "Thou are more lovely and more temperate:\"\"\"#换为单引号也可以\n",
    "print(couplet1)\n",
    "print(couplet2)\n",
    "print(couplet3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "veryveryvery\n",
      "veryveryvery\n"
     ]
    }
   ],
   "source": [
    "#字符串的加乘操作\n",
    "very_add = 'very' + 'very' + 'very'\n",
    "print(very_add)\n",
    "very_mul = 'very'*3\n",
    "print(very_mul)\n",
    "#字符串不能进行减和除的操作"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2.2 输出字符串"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Monty Python\n",
      "Monty PythonHoly Grail\n",
      "Monty Python Holy Grail\n",
      "Monty Python and the Holy Grail\n"
     ]
    }
   ],
   "source": [
    "print(monty)\n",
    "grail = 'Holy Grail'\n",
    "print(monty + grail)\n",
    "print(monty, grail)\n",
    "print(monty, \"and the\", grail)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2.3 访问单个字符串"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "M\n",
      "n\n",
      "c o l o r l e s s   g r e e n   i d e a s   s l e e p   f u r i o u s l y "
     ]
    }
   ],
   "source": [
    "print(monty[0])\n",
    "print(monty[-1])\n",
    "sent = 'colorless green ideas sleep furiously'\n",
    "#循环遍历字符串\n",
    "for char in sent:\n",
    "    print(char, end=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('e', 117092), ('t', 87996), ('a', 77916), ('o', 69326), ('n', 65617)]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['e',\n",
       " 't',\n",
       " 'a',\n",
       " 'o',\n",
       " 'n',\n",
       " 'i',\n",
       " 's',\n",
       " 'h',\n",
       " 'r',\n",
       " 'l',\n",
       " 'd',\n",
       " 'u',\n",
       " 'm',\n",
       " 'c',\n",
       " 'w',\n",
       " 'f',\n",
       " 'g',\n",
       " 'p',\n",
       " 'b',\n",
       " 'y',\n",
       " 'v',\n",
       " 'k',\n",
       " 'q',\n",
       " 'j',\n",
       " 'x',\n",
       " 'z']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 规范化为小写，并过滤非字母的字符\n",
    "from nltk.corpus import gutenberg\n",
    "raw = gutenberg.raw('melville-moby_dick.txt')\n",
    "fdist = nltk.FreqDist(ch.lower() for ch in raw if ch.isalpha())\n",
    "#按照字符出现次数进行统计\n",
    "print(fdist.most_common(5))\n",
    "[char for (char, count) in fdist.most_common()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2.4 访问子字符串"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pyth\n",
      "Monty\n",
      "Monty\n",
      "Python\n"
     ]
    }
   ],
   "source": [
    "print(monty[6:10])\n",
    "print(monty[-12:-7])\n",
    "print(monty[:5])\n",
    "print(monty[6:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found \"thing\"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "phrase = 'And now for something completely different'\n",
    "#使用in可查询字符串\n",
    "if 'thing' in phrase:\n",
    "    print('found \"thing\"')\n",
    "monty.find('Python')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2.5 更多的字符串操作"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "s.find(t) 字符串s 中包含t 的第一个索引（没找到返回-1）  \n",
    "s.rfind(t) 字符串s 中包含t 的最后一个索引（没找到返回-1）  \n",
    "s.index(t) 与s.find(t)功能类似，但没找到时引起ValueError  \n",
    "s.rindex(t) 与s.rfind(t)功能类似，但没找到时引起ValueError  \n",
    "s.join(text) 连接字符串s 与text 中的词汇  \n",
    "s.split(t) 在所有找到t 的位置将s 分割成链表（默认为空白符）  \n",
    "s.splitlines() 将s 按行分割成字符串链表  \n",
    "s.lower() 将字符串s 小写  \n",
    "s.upper() 将字符串s 大写   \n",
    "s.titlecase() 将字符串s 首字母大写  \n",
    "s.strip() 返回一个没有首尾空白字符的s 的拷贝  \n",
    "s.replace(t, u) 用u 替换s 中的t   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2.6 链表与字符串的差异\n",
    "字符串不可变  \n",
    "链表是可变"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "o\n",
      "George\n",
      "Wh\n",
      "['John', 'Paul']\n",
      "Who knows? I don't\n"
     ]
    }
   ],
   "source": [
    "query = 'Who knows?'\n",
    "beatles = ['John', 'Paul', 'George', 'Ringo']\n",
    "print(query[2])\n",
    "print(beatles[2])\n",
    "print(query[:2])\n",
    "print(beatles[:2])\n",
    "print(query + \" I don't\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "can only concatenate list (not \"str\") to list",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-30-e2b53fb6bc5e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mbeatles\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m'Brian'\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m: can only concatenate list (not \"str\") to list"
     ]
    }
   ],
   "source": [
    "#字符串和链表间不能连接\n",
    "beatles + 'Brian'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['John', 'Paul', 'George', 'Ringo', 'Brian']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "beatles + ['Brian']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.3 使用 Unicode 进行文字处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3.1 什么是 Unicode\n",
    "    Unicode是国际组织制定的可以容纳世界上所有文字和符号的字符编码方案。目前的Unicode字符分为17组编排，0x0000 至 0x10FFFF，每组称为平面（Plane），而每平面拥有65536个码位，共1114112个。  \n",
    "    Unicode 支持超过一百万种字符。每个字符分配一个编号，称为编码点。在Python 中，编码点写作\\uXXXX 的形式，其中XXXX 是四位十六进制形式数。  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3.2 从文件中提取已编码文本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pruska Biblioteka Państwowa. Jej dawne zbiory znane pod nazwą\n",
      "\"Berlinka\" to skarb kultury i sztuki niemieckiej. Przewiezione przez\n",
      "Niemców pod koniec II wojny światowej na Dolny Śląsk, zostały\n",
      "odnalezione po 1945 r. na terytorium Polski. Trafiły do Biblioteki\n",
      "Jagiellońskiej w Krakowie, obejmują ponad 500 tys. zabytkowych\n",
      "archiwaliów, m.in. manuskrypty Goethego, Mozarta, Beethovena, Bacha.\n"
     ]
    }
   ],
   "source": [
    "path = nltk.data.find('corpora/unicode_samples/polish-lat2.txt')\n",
    "#用编码'latin2'打开波兰编码文件\n",
    "f = open(path, encoding='latin2')\n",
    "for line in f:\n",
    "    line = line.strip()\n",
    "    print(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'Pruska Biblioteka Pa\\\\u0144stwowa. Jej dawne zbiory znane pod nazw\\\\u0105'\n",
      "b'\"Berlinka\" to skarb kultury i sztuki niemieckiej. Przewiezione przez'\n",
      "b'Niemc\\\\xf3w pod koniec II wojny \\\\u015bwiatowej na Dolny \\\\u015al\\\\u0105sk, zosta\\\\u0142y'\n",
      "b'odnalezione po 1945 r. na terytorium Polski. Trafi\\\\u0142y do Biblioteki'\n",
      "b'Jagiello\\\\u0144skiej w Krakowie, obejmuj\\\\u0105 ponad 500 tys. zabytkowych'\n",
      "b'archiwali\\\\xf3w, m.in. manuskrypty Goethego, Mozarta, Beethovena, Bacha.'\n"
     ]
    }
   ],
   "source": [
    "f = open(path, encoding='latin2')\n",
    "for line in f:\n",
    "    line = line.strip()\n",
    "    print(line.encode('unicode_escape'))#将所有非ascii字符转换为它们的两位数\\xXX和四位数字\\uXXXX表示:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "324"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ord('ń')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ń'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nacute = '\\u0144'\n",
    "nacute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'\\xc5\\x84'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nacute.encode('utf8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3.3 在 Python中使用本地编码"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "需要在文件的第一行或第二行中包含字符串：'# -*- coding: <coding>-*-'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.4 使用正则表达式检测词组搭配\n",
    "正则表达式通常被用来检索、替换那些符合某个模式(规则)的文本   \n",
    "需要使用import re 导入re 函数库"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.4.1使用基本的元字符"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abaissed',\n",
       " 'abandoned',\n",
       " 'abased',\n",
       " 'abashed',\n",
       " 'abatised',\n",
       " 'abed',\n",
       " 'aborted',\n",
       " 'abridged',\n",
       " 'abscessed',\n",
       " 'absconded',\n",
       " 'absorbed',\n",
       " 'abstracted',\n",
       " 'abstricted',\n",
       " 'accelerated',\n",
       " 'accepted',\n",
       " 'accidented',\n",
       " 'accoladed',\n",
       " 'accolated',\n",
       " 'accomplished',\n",
       " 'accosted',\n",
       " 'accredited',\n",
       " 'accursed',\n",
       " 'accused',\n",
       " 'accustomed',\n",
       " 'acetated',\n",
       " 'acheweed',\n",
       " 'aciculated',\n",
       " 'aciliated',\n",
       " 'acknowledged',\n",
       " 'acorned',\n",
       " 'acquainted',\n",
       " 'acquired',\n",
       " 'acquisited',\n",
       " 'acred',\n",
       " 'aculeated',\n",
       " 'addebted',\n",
       " 'added',\n",
       " 'addicted',\n",
       " 'addlebrained',\n",
       " 'addleheaded',\n",
       " 'addlepated',\n",
       " 'addorsed',\n",
       " 'adempted',\n",
       " 'adfected',\n",
       " 'adjoined',\n",
       " 'admired',\n",
       " 'admitted',\n",
       " 'adnexed',\n",
       " 'adopted',\n",
       " 'adossed',\n",
       " 'adreamed',\n",
       " 'adscripted',\n",
       " 'aduncated',\n",
       " 'advanced',\n",
       " 'advised',\n",
       " 'aeried',\n",
       " 'aethered',\n",
       " 'afeared',\n",
       " 'affected',\n",
       " 'affectioned',\n",
       " 'affined',\n",
       " 'afflicted',\n",
       " 'affricated',\n",
       " 'affrighted',\n",
       " 'affronted',\n",
       " 'aforenamed',\n",
       " 'afterfeed',\n",
       " 'aftershafted',\n",
       " 'afterthoughted',\n",
       " 'afterwitted',\n",
       " 'agazed',\n",
       " 'aged',\n",
       " 'agglomerated',\n",
       " 'aggrieved',\n",
       " 'agminated',\n",
       " 'agnamed',\n",
       " 'agonied',\n",
       " 'agreed',\n",
       " 'agueweed',\n",
       " 'ahungered',\n",
       " 'aiguilletted',\n",
       " 'ailweed',\n",
       " 'airbrained',\n",
       " 'airified',\n",
       " 'aiseweed',\n",
       " 'aisled',\n",
       " 'alarmed',\n",
       " 'alated',\n",
       " 'alimonied',\n",
       " 'aliped',\n",
       " 'alleyed',\n",
       " 'allied',\n",
       " 'alligatored',\n",
       " 'allseed',\n",
       " 'almsdeed',\n",
       " 'aloed',\n",
       " 'altared',\n",
       " 'alveolated',\n",
       " 'amazed',\n",
       " 'ameed',\n",
       " 'amiced',\n",
       " 'amphitheatered',\n",
       " 'ampullated',\n",
       " 'amused',\n",
       " 'anchored',\n",
       " 'angled',\n",
       " 'anguiped',\n",
       " 'anguished',\n",
       " 'angulated',\n",
       " 'angulinerved',\n",
       " 'anhungered',\n",
       " 'animated',\n",
       " 'aniseed',\n",
       " 'annodated',\n",
       " 'annulated',\n",
       " 'anomaliped',\n",
       " 'anserated',\n",
       " 'anteflected',\n",
       " 'anteflexed',\n",
       " 'antimoniated',\n",
       " 'antimoniureted',\n",
       " 'antimoniuretted',\n",
       " 'antiquated',\n",
       " 'antired',\n",
       " 'antiweed',\n",
       " 'antlered',\n",
       " 'apertured',\n",
       " 'apexed',\n",
       " 'apicifixed',\n",
       " 'apiculated',\n",
       " 'apocopated',\n",
       " 'apostrophied',\n",
       " 'appearanced',\n",
       " 'appellatived',\n",
       " 'appendaged',\n",
       " 'appendiculated',\n",
       " 'applied',\n",
       " 'appressed',\n",
       " 'aralkylated',\n",
       " 'arbored',\n",
       " 'arched',\n",
       " 'architraved',\n",
       " 'arcked',\n",
       " 'arcuated',\n",
       " 'ared',\n",
       " 'areolated',\n",
       " 'ariled',\n",
       " 'arillated',\n",
       " 'armchaired',\n",
       " 'armed',\n",
       " 'armied',\n",
       " 'armillated',\n",
       " 'armored',\n",
       " 'armoried',\n",
       " 'arpeggiated',\n",
       " 'arpeggioed',\n",
       " 'arrased',\n",
       " 'arrowed',\n",
       " 'arrowheaded',\n",
       " 'arrowweed',\n",
       " 'arseneted',\n",
       " 'arsenetted',\n",
       " 'arseniureted',\n",
       " 'articled',\n",
       " 'articulated',\n",
       " 'ashamed',\n",
       " 'ashlared',\n",
       " 'ashweed',\n",
       " 'aspersed',\n",
       " 'asphyxied',\n",
       " 'assented',\n",
       " 'assessed',\n",
       " 'assigned',\n",
       " 'assistanted',\n",
       " 'associated',\n",
       " 'assonanced',\n",
       " 'assorted',\n",
       " 'assumed',\n",
       " 'assured',\n",
       " 'asteriated',\n",
       " 'astonied',\n",
       " 'aswooned',\n",
       " 'atrophiated',\n",
       " 'atrophied',\n",
       " 'attached',\n",
       " 'attired',\n",
       " 'attrited',\n",
       " 'augmented',\n",
       " 'aurated',\n",
       " 'auricled',\n",
       " 'auriculated',\n",
       " 'authorized',\n",
       " 'autoinhibited',\n",
       " 'autosensitized',\n",
       " 'autosled',\n",
       " 'averted',\n",
       " 'avowed',\n",
       " 'awearied',\n",
       " 'awned',\n",
       " 'awninged',\n",
       " 'axed',\n",
       " 'axhammered',\n",
       " 'axised',\n",
       " 'axled',\n",
       " 'axseed',\n",
       " 'axweed',\n",
       " 'azoted',\n",
       " 'azured',\n",
       " 'babied',\n",
       " 'babished',\n",
       " 'babyfied',\n",
       " 'baccated',\n",
       " 'backboned',\n",
       " 'backed',\n",
       " 'backhanded',\n",
       " 'backwatered',\n",
       " 'baconweed',\n",
       " 'badgerweed',\n",
       " 'bagged',\n",
       " 'bagwigged',\n",
       " 'baked',\n",
       " 'balanced',\n",
       " 'balconied',\n",
       " 'baldachined',\n",
       " 'baldricked',\n",
       " 'balled',\n",
       " 'ballweed',\n",
       " 'balsamweed',\n",
       " 'balustered',\n",
       " 'balustraded',\n",
       " 'bandannaed',\n",
       " 'banded',\n",
       " 'bandoleered',\n",
       " 'bangled',\n",
       " 'banked',\n",
       " 'bankweed',\n",
       " 'bannered',\n",
       " 'barbated',\n",
       " 'barbed',\n",
       " 'barebacked',\n",
       " 'bareboned',\n",
       " 'barefaced',\n",
       " 'barefooted',\n",
       " 'barehanded',\n",
       " 'bareheaded',\n",
       " 'barelegged',\n",
       " 'barenecked',\n",
       " 'barmybrained',\n",
       " 'barred',\n",
       " 'barreled',\n",
       " 'bartizaned',\n",
       " 'basebred',\n",
       " 'based',\n",
       " 'basehearted',\n",
       " 'basifixed',\n",
       " 'basilweed',\n",
       " 'basined',\n",
       " 'basinerved',\n",
       " 'basqued',\n",
       " 'bastioned',\n",
       " 'bated',\n",
       " 'bathroomed',\n",
       " 'battered',\n",
       " 'batteried',\n",
       " 'battled',\n",
       " 'battlemented',\n",
       " 'bayed',\n",
       " 'bayoneted',\n",
       " 'beached',\n",
       " 'beaded',\n",
       " 'beaked',\n",
       " 'bealtared',\n",
       " 'beamed',\n",
       " 'beanweed',\n",
       " 'beaproned',\n",
       " 'bearded',\n",
       " 'beautied',\n",
       " 'beavered',\n",
       " 'beballed',\n",
       " 'bebannered',\n",
       " 'bebed',\n",
       " 'bebelted',\n",
       " 'bebled',\n",
       " 'bebothered',\n",
       " 'bebouldered',\n",
       " 'bebuttoned',\n",
       " 'becassocked',\n",
       " 'bechained',\n",
       " 'bechignoned',\n",
       " 'becircled',\n",
       " 'becoiffed',\n",
       " 'becombed',\n",
       " 'becousined',\n",
       " 'becrinolined',\n",
       " 'becuffed',\n",
       " 'becurtained',\n",
       " 'becushioned',\n",
       " 'bed',\n",
       " 'bedaggered',\n",
       " 'bedangled',\n",
       " 'bedded',\n",
       " 'bediademed',\n",
       " 'bediamonded',\n",
       " 'beedged',\n",
       " 'beefheaded',\n",
       " 'beeheaded',\n",
       " 'beeswinged',\n",
       " 'beetled',\n",
       " 'beetleheaded',\n",
       " 'beetleweed',\n",
       " 'beeweed',\n",
       " 'befamilied',\n",
       " 'befanned',\n",
       " 'befathered',\n",
       " 'beferned',\n",
       " 'befetished',\n",
       " 'befezzed',\n",
       " 'befilleted',\n",
       " 'befilmed',\n",
       " 'beforested',\n",
       " 'befountained',\n",
       " 'befrocked',\n",
       " 'befrogged',\n",
       " 'befurbelowed',\n",
       " 'befurred',\n",
       " 'begabled',\n",
       " 'begarlanded',\n",
       " 'begartered',\n",
       " 'beggarweed',\n",
       " 'beglobed',\n",
       " 'begoggled',\n",
       " 'begowned',\n",
       " 'behatted',\n",
       " 'behaviored',\n",
       " 'beheadlined',\n",
       " 'behooped',\n",
       " 'beinked',\n",
       " 'bekilted',\n",
       " 'beknived',\n",
       " 'beknotted',\n",
       " 'belaced',\n",
       " 'belated',\n",
       " 'belatticed',\n",
       " 'belavendered',\n",
       " 'beledgered',\n",
       " 'belfried',\n",
       " 'beliked',\n",
       " 'belimousined',\n",
       " 'belled',\n",
       " 'bellied',\n",
       " 'bellmouthed',\n",
       " 'bellweed',\n",
       " 'beloved',\n",
       " 'belozenged',\n",
       " 'belted',\n",
       " 'bemazed',\n",
       " 'bemedaled',\n",
       " 'bemedalled',\n",
       " 'bemitered',\n",
       " 'bemitred',\n",
       " 'bemused',\n",
       " 'bemuslined',\n",
       " 'bended',\n",
       " 'beneaped',\n",
       " 'beneficed',\n",
       " 'beneighbored',\n",
       " 'benempted',\n",
       " 'benighted',\n",
       " 'bennetweed',\n",
       " 'benumbed',\n",
       " 'benweed',\n",
       " 'benzoated',\n",
       " 'benzoinated',\n",
       " 'bepastured',\n",
       " 'bepatched',\n",
       " 'beperiwigged',\n",
       " 'bepewed',\n",
       " 'bepillared',\n",
       " 'bepistoled',\n",
       " 'beplaided',\n",
       " 'beplumed',\n",
       " 'beribanded',\n",
       " 'beribboned',\n",
       " 'beringed',\n",
       " 'beringleted',\n",
       " 'berobed',\n",
       " 'berouged',\n",
       " 'berried',\n",
       " 'berthed',\n",
       " 'beruffed',\n",
       " 'beruffled',\n",
       " 'beshawled',\n",
       " 'besieged',\n",
       " 'beslushed',\n",
       " 'besotted',\n",
       " 'bespecked',\n",
       " 'bespectacled',\n",
       " 'besped',\n",
       " 'bespeed',\n",
       " 'bespelled',\n",
       " 'bespurred',\n",
       " 'bestatued',\n",
       " 'bestayed',\n",
       " 'bestrapped',\n",
       " 'bestubbled',\n",
       " 'besweatered',\n",
       " 'betattered',\n",
       " 'betaxed',\n",
       " 'betowered',\n",
       " 'betrothed',\n",
       " 'betrousered',\n",
       " 'betted',\n",
       " 'betuckered',\n",
       " 'beturbaned',\n",
       " 'betusked',\n",
       " 'betutored',\n",
       " 'betwattled',\n",
       " 'beuniformed',\n",
       " 'beveled',\n",
       " 'bevelled',\n",
       " 'bevesseled',\n",
       " 'bevesselled',\n",
       " 'bevined',\n",
       " 'bevoiled',\n",
       " 'bewaitered',\n",
       " 'bewhiskered',\n",
       " 'bewigged',\n",
       " 'bewildered',\n",
       " 'bewinged',\n",
       " 'bewired',\n",
       " 'bewrathed',\n",
       " 'biangulated',\n",
       " 'biarcuated',\n",
       " 'biarticulated',\n",
       " 'bicarbureted',\n",
       " 'biciliated',\n",
       " 'bicolored',\n",
       " 'bicorned',\n",
       " 'bidented',\n",
       " 'bifanged',\n",
       " 'bifidated',\n",
       " 'biflected',\n",
       " 'biforked',\n",
       " 'biformed',\n",
       " 'bifronted',\n",
       " 'bifurcated',\n",
       " 'bigeminated',\n",
       " 'bighearted',\n",
       " 'bigmouthed',\n",
       " 'bigoted',\n",
       " 'bigwigged',\n",
       " 'bilamellated',\n",
       " 'bilaminated',\n",
       " 'billed',\n",
       " 'bilobated',\n",
       " 'bilobed',\n",
       " 'bilsted',\n",
       " 'bimaculated',\n",
       " 'bimotored',\n",
       " 'bindweed',\n",
       " 'bineweed',\n",
       " 'binominated',\n",
       " 'binucleated',\n",
       " 'biparted',\n",
       " 'bipectinated',\n",
       " 'biped',\n",
       " 'bipennated',\n",
       " 'bipinnated',\n",
       " 'bipinnatiparted',\n",
       " 'bipinnatisected',\n",
       " 'biradiated',\n",
       " 'birdmouthed',\n",
       " 'birdseed',\n",
       " 'birdweed',\n",
       " 'birostrated',\n",
       " 'birthbed',\n",
       " 'bisexed',\n",
       " 'bishopweed',\n",
       " 'bistered',\n",
       " 'bistipuled',\n",
       " 'bisubstituted',\n",
       " 'bitted',\n",
       " 'bitterhearted',\n",
       " 'bitterweed',\n",
       " 'bituberculated',\n",
       " 'bitumed',\n",
       " 'bivalved',\n",
       " 'bivaulted',\n",
       " 'bivocalized',\n",
       " 'blackhearted',\n",
       " 'blackseed',\n",
       " 'blackshirted',\n",
       " 'bladderseed',\n",
       " 'bladderweed',\n",
       " 'bladed',\n",
       " 'blakeberyed',\n",
       " 'blamed',\n",
       " 'blanked',\n",
       " 'blanketed',\n",
       " 'blanketweed',\n",
       " 'blasted',\n",
       " 'bleached',\n",
       " 'bleared',\n",
       " 'bleed',\n",
       " 'blended',\n",
       " 'blessed',\n",
       " 'blighted',\n",
       " 'blinded',\n",
       " 'blindfolded',\n",
       " 'blindweed',\n",
       " 'blinked',\n",
       " 'blinkered',\n",
       " 'blistered',\n",
       " 'blisterweed',\n",
       " 'blithehearted',\n",
       " 'bloated',\n",
       " 'blobbed',\n",
       " 'blocked',\n",
       " 'blockheaded',\n",
       " 'blooded',\n",
       " 'bloodied',\n",
       " 'bloodshed',\n",
       " 'bloodstained',\n",
       " 'bloodweed',\n",
       " 'blossomed',\n",
       " 'blotched',\n",
       " 'bloused',\n",
       " 'blowzed',\n",
       " 'bludgeoned',\n",
       " 'bluebelled',\n",
       " 'bluehearted',\n",
       " 'blueweed',\n",
       " 'blunderheaded',\n",
       " 'blunthearted',\n",
       " 'blurred',\n",
       " 'bobbed',\n",
       " 'bobsled',\n",
       " 'bobtailed',\n",
       " 'bodiced',\n",
       " 'bodied',\n",
       " 'boiled',\n",
       " 'boldhearted',\n",
       " 'bolectioned',\n",
       " 'boled',\n",
       " 'boleweed',\n",
       " 'bolled',\n",
       " 'bombed',\n",
       " 'bonded',\n",
       " 'boned',\n",
       " 'boneheaded',\n",
       " 'bonneted',\n",
       " 'booked',\n",
       " 'booted',\n",
       " 'bootied',\n",
       " 'boozed',\n",
       " 'bordered',\n",
       " 'bordured',\n",
       " 'bosomed',\n",
       " 'bossed',\n",
       " 'bosselated',\n",
       " 'botched',\n",
       " 'botherheaded',\n",
       " 'bothsided',\n",
       " 'bottled',\n",
       " 'bottomed',\n",
       " 'boughed',\n",
       " 'bounded',\n",
       " 'bountied',\n",
       " 'bowed',\n",
       " 'boweled',\n",
       " 'bowlegged',\n",
       " 'bowstringed',\n",
       " 'braced',\n",
       " 'braceleted',\n",
       " 'brackened',\n",
       " 'bracted',\n",
       " 'braided',\n",
       " 'brambled',\n",
       " 'branched',\n",
       " 'branded',\n",
       " 'brandied',\n",
       " 'brangled',\n",
       " 'bravehearted',\n",
       " 'brawned',\n",
       " 'brazenfaced',\n",
       " 'breasted',\n",
       " 'breastweed',\n",
       " 'breathed',\n",
       " 'brecciated',\n",
       " 'bred',\n",
       " 'breeched',\n",
       " 'breed',\n",
       " 'breviped',\n",
       " 'bridebed',\n",
       " 'brideweed',\n",
       " 'bridged',\n",
       " 'bridled',\n",
       " 'briered',\n",
       " 'brimmed',\n",
       " 'bristled',\n",
       " 'broadhearted',\n",
       " 'brocaded',\n",
       " 'brocked',\n",
       " 'brokenhearted',\n",
       " 'bromoiodized',\n",
       " 'bronzed',\n",
       " 'brooked',\n",
       " 'brookweed',\n",
       " 'broomweed',\n",
       " 'broozled',\n",
       " 'browed',\n",
       " 'brownweed',\n",
       " 'bruckled',\n",
       " 'brushed',\n",
       " 'buboed',\n",
       " 'bucked',\n",
       " 'buckled',\n",
       " 'buckskinned',\n",
       " 'buffed',\n",
       " 'bugled',\n",
       " 'bugleweed',\n",
       " 'bugseed',\n",
       " 'bugweed',\n",
       " 'bulbed',\n",
       " 'bulked',\n",
       " 'bulkheaded',\n",
       " 'bullated',\n",
       " 'bulldogged',\n",
       " 'bulleted',\n",
       " 'bulletheaded',\n",
       " 'bullheaded',\n",
       " 'bullweed',\n",
       " 'bummed',\n",
       " 'bundlerooted',\n",
       " 'bundweed',\n",
       " 'bunted',\n",
       " 'buried',\n",
       " 'burled',\n",
       " 'burned',\n",
       " 'burnoosed',\n",
       " 'burntweed',\n",
       " 'burred',\n",
       " 'burroweed',\n",
       " 'burseed',\n",
       " 'burweed',\n",
       " 'bushed',\n",
       " 'busied',\n",
       " 'busked',\n",
       " 'buskined',\n",
       " 'busted',\n",
       " 'bustled',\n",
       " 'busybodied',\n",
       " 'buttered',\n",
       " 'butterfingered',\n",
       " 'butterweed',\n",
       " 'butteryfingered',\n",
       " 'buttocked',\n",
       " 'buttoned',\n",
       " 'buttonweed',\n",
       " 'cabled',\n",
       " 'caboshed',\n",
       " 'caddiced',\n",
       " 'caddised',\n",
       " 'cadenced',\n",
       " 'cadweed',\n",
       " 'caftaned',\n",
       " 'caged',\n",
       " 'cairned',\n",
       " 'caissoned',\n",
       " 'calced',\n",
       " 'calcified',\n",
       " 'calcined',\n",
       " 'calculated',\n",
       " 'calibered',\n",
       " 'calicoed',\n",
       " 'caligated',\n",
       " 'calpacked',\n",
       " 'calved',\n",
       " 'calycled',\n",
       " 'calyculated',\n",
       " 'camailed',\n",
       " 'camerated',\n",
       " 'cammed',\n",
       " 'campanulated',\n",
       " 'campshed',\n",
       " 'camused',\n",
       " 'canaliculated',\n",
       " 'cancellated',\n",
       " 'cancered',\n",
       " 'cancerweed',\n",
       " 'candied',\n",
       " 'candlelighted',\n",
       " 'candlesticked',\n",
       " 'candyweed',\n",
       " 'canioned',\n",
       " 'cankered',\n",
       " 'cankerweed',\n",
       " 'canned',\n",
       " 'cannelated',\n",
       " 'cannelured',\n",
       " 'cannoned',\n",
       " 'cannulated',\n",
       " 'canted',\n",
       " 'cantilevered',\n",
       " 'cantoned',\n",
       " 'cantred',\n",
       " 'caped',\n",
       " 'capernoited',\n",
       " 'capeweed',\n",
       " 'capitaled',\n",
       " 'capitated',\n",
       " 'capped',\n",
       " 'capriped',\n",
       " 'capsulated',\n",
       " 'capuched',\n",
       " 'carapaced',\n",
       " 'carbolated',\n",
       " 'carboyed',\n",
       " 'carbuncled',\n",
       " 'carcaneted',\n",
       " 'carded',\n",
       " 'carinated',\n",
       " 'carkled',\n",
       " 'carnaged',\n",
       " 'carnationed',\n",
       " 'carpetweed',\n",
       " 'carried',\n",
       " 'carrotweed',\n",
       " 'carucated',\n",
       " 'carunculated',\n",
       " 'cased',\n",
       " 'casemated',\n",
       " 'casemented',\n",
       " 'caseweed',\n",
       " 'casqued',\n",
       " 'castellated',\n",
       " 'castled',\n",
       " 'castorized',\n",
       " 'catamited',\n",
       " 'cataracted',\n",
       " 'catarrhed',\n",
       " 'catchweed',\n",
       " 'catenated',\n",
       " 'caterpillared',\n",
       " 'catfaced',\n",
       " 'catfooted',\n",
       " 'cathedraled',\n",
       " 'caudated',\n",
       " 'caverned',\n",
       " 'cavitied',\n",
       " 'cayenned',\n",
       " 'cedared',\n",
       " 'ceilinged',\n",
       " 'celebrated',\n",
       " 'cellated',\n",
       " 'celled',\n",
       " 'cellulated',\n",
       " 'celluloided',\n",
       " 'centered',\n",
       " 'centriffed',\n",
       " 'centuried',\n",
       " 'cerated',\n",
       " 'cered',\n",
       " 'certified',\n",
       " 'chafeweed',\n",
       " 'chaffseed',\n",
       " 'chaffweed',\n",
       " 'chafted',\n",
       " 'chained',\n",
       " 'chaliced',\n",
       " 'chambered',\n",
       " 'chamberleted',\n",
       " 'chamberletted',\n",
       " 'chanceled',\n",
       " 'channeled',\n",
       " 'channelled',\n",
       " 'chaped',\n",
       " 'chapleted',\n",
       " 'chapournetted',\n",
       " 'chapped',\n",
       " 'charioted',\n",
       " 'charqued',\n",
       " 'chartered',\n",
       " 'chasmed',\n",
       " 'chasteweed',\n",
       " 'chasubled',\n",
       " 'checked',\n",
       " 'checkered',\n",
       " 'checkrowed',\n",
       " 'cheered',\n",
       " 'cheliped',\n",
       " 'cherried',\n",
       " 'chickenbreasted',\n",
       " 'chickenhearted',\n",
       " 'chickenweed',\n",
       " 'chickweed',\n",
       " 'chicqued',\n",
       " 'chiggerweed',\n",
       " 'chignoned',\n",
       " 'childbed',\n",
       " 'childed',\n",
       " 'chilled',\n",
       " 'chined',\n",
       " 'chinned',\n",
       " 'chipped',\n",
       " 'chiseled',\n",
       " 'chitinized',\n",
       " 'chokered',\n",
       " 'chokeweed',\n",
       " 'cholterheaded',\n",
       " 'chopped',\n",
       " 'choppered',\n",
       " 'chorded',\n",
       " 'chowderheaded',\n",
       " 'christened',\n",
       " 'chubbed',\n",
       " 'chuckleheaded',\n",
       " 'churchified',\n",
       " 'churled',\n",
       " 'ciliated',\n",
       " 'cingulated',\n",
       " 'cinnamoned',\n",
       " 'cinquefoiled',\n",
       " 'circled',\n",
       " 'circumscribed',\n",
       " 'circumstanced',\n",
       " 'cirrated',\n",
       " 'cirrhosed',\n",
       " 'cirriped',\n",
       " 'cisted',\n",
       " 'citied',\n",
       " 'citified',\n",
       " 'citrated',\n",
       " 'civilized',\n",
       " 'clammed',\n",
       " 'clammyweed',\n",
       " 'clanned',\n",
       " 'clapped',\n",
       " 'classed',\n",
       " 'classified',\n",
       " 'clavated',\n",
       " 'clavellated',\n",
       " 'clawed',\n",
       " 'claybrained',\n",
       " 'clayweed',\n",
       " 'cleaded',\n",
       " 'cleanhanded',\n",
       " 'cleanhearted',\n",
       " 'clearheaded',\n",
       " 'clearhearted',\n",
       " 'clearweed',\n",
       " 'cled',\n",
       " 'cleeked',\n",
       " 'clefted',\n",
       " 'clerestoried',\n",
       " 'cliented',\n",
       " 'cliffed',\n",
       " 'cliffweed',\n",
       " 'clipped',\n",
       " 'cloaked',\n",
       " 'clocked',\n",
       " 'clodpated',\n",
       " 'cloistered',\n",
       " 'closed',\n",
       " 'closefisted',\n",
       " 'closehanded',\n",
       " 'closehearted',\n",
       " 'closemouthed',\n",
       " 'clotweed',\n",
       " 'clouded',\n",
       " 'clouted',\n",
       " 'clovered',\n",
       " 'clubbed',\n",
       " 'clubfisted',\n",
       " 'clubfooted',\n",
       " 'clubweed',\n",
       " 'clustered',\n",
       " 'coaged',\n",
       " 'coaggregated',\n",
       " 'coated',\n",
       " 'coattailed',\n",
       " 'cobbed',\n",
       " 'cocashweed',\n",
       " 'cochleated',\n",
       " 'cockaded',\n",
       " 'cocked',\n",
       " 'cockeyed',\n",
       " 'cockled',\n",
       " 'cockneybred',\n",
       " 'cockscombed',\n",
       " 'cockweed',\n",
       " 'codheaded',\n",
       " 'coed',\n",
       " 'coelongated',\n",
       " 'coembedded',\n",
       " 'coequated',\n",
       " 'coexpanded',\n",
       " 'coffeeweed',\n",
       " 'cogged',\n",
       " 'coifed',\n",
       " 'coiled',\n",
       " 'coldhearted',\n",
       " 'coleseed',\n",
       " 'colicweed',\n",
       " 'collared',\n",
       " 'collected',\n",
       " 'collied',\n",
       " 'colloped',\n",
       " 'colonnaded',\n",
       " 'colored',\n",
       " 'columnated',\n",
       " 'columned',\n",
       " 'combed',\n",
       " 'combined',\n",
       " 'compacted',\n",
       " 'complected',\n",
       " 'complexioned',\n",
       " 'complicated',\n",
       " 'componed',\n",
       " 'componented',\n",
       " 'composed',\n",
       " 'compressed',\n",
       " 'comprised',\n",
       " 'compulsed',\n",
       " 'conamed',\n",
       " 'concamerated',\n",
       " 'concealed',\n",
       " 'conceded',\n",
       " 'conceited',\n",
       " 'concentrated',\n",
       " 'concerned',\n",
       " 'concerted',\n",
       " 'conched',\n",
       " 'conchyliated',\n",
       " 'condemned',\n",
       " 'condensed',\n",
       " 'conditioned',\n",
       " 'conduplicated',\n",
       " 'coned',\n",
       " 'confated',\n",
       " 'conferted',\n",
       " 'confined',\n",
       " 'confirmed',\n",
       " 'conflated',\n",
       " 'confounded',\n",
       " 'confused',\n",
       " 'congested',\n",
       " 'conjoined',\n",
       " 'conjugated',\n",
       " 'connected',\n",
       " 'conred',\n",
       " 'consecrated',\n",
       " 'considered',\n",
       " 'consolidated',\n",
       " 'constrained',\n",
       " 'constricted',\n",
       " 'consumpted',\n",
       " 'contagioned',\n",
       " 'contented',\n",
       " 'contextured',\n",
       " 'continued',\n",
       " 'contorted',\n",
       " 'contortioned',\n",
       " 'contracted',\n",
       " 'contractured',\n",
       " 'contusioned',\n",
       " 'converted',\n",
       " 'convexed',\n",
       " 'convinced',\n",
       " 'convoluted',\n",
       " 'coolheaded',\n",
       " 'coolweed',\n",
       " 'copied',\n",
       " 'copleased',\n",
       " 'copped',\n",
       " 'coppernosed',\n",
       " 'copperytailed',\n",
       " 'coppiced',\n",
       " 'coppled',\n",
       " 'copsewooded',\n",
       " 'copygraphed',\n",
       " 'coraled',\n",
       " 'corded',\n",
       " 'corduroyed',\n",
       " 'cored',\n",
       " 'coreflexed',\n",
       " 'corked',\n",
       " 'cornered',\n",
       " 'cornified',\n",
       " 'cornuated',\n",
       " 'cornuted',\n",
       " 'corollated',\n",
       " 'coronaled',\n",
       " 'coronated',\n",
       " 'coroneted',\n",
       " 'coronetted',\n",
       " 'corpusculated',\n",
       " 'corrected',\n",
       " 'correlated',\n",
       " 'corridored',\n",
       " ...]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re,nltk\n",
    "wordlist = [w for w in nltk.corpus.words.words('en') if w.islower()]\n",
    "# 使用正则表达式«ed$»查找以ed 结尾的词汇\n",
    "[w for w in wordlist if re.search('ed$', w)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abjectly',\n",
       " 'adjuster',\n",
       " 'dejected',\n",
       " 'dejectly',\n",
       " 'injector',\n",
       " 'majestic',\n",
       " 'objectee',\n",
       " 'objector',\n",
       " 'rejecter',\n",
       " 'rejector',\n",
       " 'unjilted',\n",
       " 'unjolted',\n",
       " 'unjustly']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#通配符“.”匹配任何单个字符\n",
    "[w for w in wordlist if re.search('^..j..t..$', w)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.4.2 范围与闭包\n",
    "re.search匹配整个字符串，直到找到一个匹配  \n",
    "· 通配符，匹配所有字符  \n",
    "^abc 匹配以abc 开始的字符串  \n",
    "abc$ 匹配以abc 结尾的字符串  \n",
    "[abc] 匹配字符集合中的一个  \n",
    "[A-Z0-9] 匹配字符一个范围  \n",
    "ed|ing|s 匹配指定的一个字符串（析取）  \n",
    "“*” 前面的项目零个或多个，如a*, [a-z]* (也叫Kleene 闭包)  \n",
    "“+” 前面的项目1 个或多个，如a+, [a-z]+  \n",
    "“?” 前面的项目零个或1 个（即：可选）如：a?, [a-z]?  \n",
    "{n} 重复n 次，n 为非负整数  \n",
    "{n,} 至少重复n 次  \n",
    "{,n} 重复不多于n 次  \n",
    "{m,n} 至少重复m 次不多于n 次  \n",
    "a(b|c)+ 括号表示操作符的范围  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['gold', 'golf', 'hold', 'hole']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[w for w in wordlist if re.search('^[ghi][mno][jlk][def]$', w)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['miiiiiiiiiiiiinnnnnnnnnnneeeeeeeeee',\n",
       " 'miiiiiinnnnnnnnnneeeeeeee',\n",
       " 'mine',\n",
       " 'mmmmmmmmiiiiiiiiinnnnnnnnneeeeeeee']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chat_words = sorted(set(w for w in nltk.corpus.nps_chat.words()))\n",
    "[w for w in chat_words if re.search('^m+i+n+e+$', w)]\n",
    "#“+”表示的是“前面字符的一个或多个实例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a',\n",
       " 'aaaaaaaaaaaaaaaaa',\n",
       " 'aaahhhh',\n",
       " 'ah',\n",
       " 'ahah',\n",
       " 'ahahah',\n",
       " 'ahh',\n",
       " 'ahhahahaha',\n",
       " 'ahhh',\n",
       " 'ahhhh',\n",
       " 'ahhhhhh',\n",
       " 'ahhhhhhhhhhhhhh',\n",
       " 'h',\n",
       " 'ha',\n",
       " 'haaa',\n",
       " 'hah',\n",
       " 'haha',\n",
       " 'hahaaa',\n",
       " 'hahah',\n",
       " 'hahaha',\n",
       " 'hahahaa',\n",
       " 'hahahah',\n",
       " 'hahahaha',\n",
       " 'hahahahaaa',\n",
       " 'hahahahahaha',\n",
       " 'hahahahahahaha',\n",
       " 'hahahahahahahahahahahahahahahaha',\n",
       " 'hahahhahah',\n",
       " 'hahhahahaha']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[w for w in chat_words if re.search('^[ha]+$', w)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "wsj = sorted(set(nltk.corpus.treebank.words()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['0.0085',\n",
       " '0.05',\n",
       " '0.1',\n",
       " '0.16',\n",
       " '0.2',\n",
       " '0.25',\n",
       " '0.28',\n",
       " '0.3',\n",
       " '0.4',\n",
       " '0.5',\n",
       " '0.50',\n",
       " '0.54',\n",
       " '0.56',\n",
       " '0.60',\n",
       " '0.7',\n",
       " '0.82',\n",
       " '0.84',\n",
       " '0.9',\n",
       " '0.95',\n",
       " '0.99',\n",
       " '1.01',\n",
       " '1.1',\n",
       " '1.125',\n",
       " '1.14',\n",
       " '1.1650',\n",
       " '1.17',\n",
       " '1.18',\n",
       " '1.19',\n",
       " '1.2',\n",
       " '1.20',\n",
       " '1.24',\n",
       " '1.25',\n",
       " '1.26',\n",
       " '1.28',\n",
       " '1.35',\n",
       " '1.39',\n",
       " '1.4',\n",
       " '1.457',\n",
       " '1.46',\n",
       " '1.49',\n",
       " '1.5',\n",
       " '1.50',\n",
       " '1.55',\n",
       " '1.56',\n",
       " '1.5755',\n",
       " '1.5805',\n",
       " '1.6',\n",
       " '1.61',\n",
       " '1.637',\n",
       " '1.64',\n",
       " '1.65',\n",
       " '1.7',\n",
       " '1.75',\n",
       " '1.76',\n",
       " '1.8',\n",
       " '1.82',\n",
       " '1.8415',\n",
       " '1.85',\n",
       " '1.8500',\n",
       " '1.9',\n",
       " '1.916',\n",
       " '1.92',\n",
       " '10.19',\n",
       " '10.2',\n",
       " '10.5',\n",
       " '107.03',\n",
       " '107.9',\n",
       " '109.73',\n",
       " '11.10',\n",
       " '11.5',\n",
       " '11.57',\n",
       " '11.6',\n",
       " '11.72',\n",
       " '11.95',\n",
       " '112.9',\n",
       " '113.2',\n",
       " '116.3',\n",
       " '116.4',\n",
       " '116.7',\n",
       " '116.9',\n",
       " '118.6',\n",
       " '12.09',\n",
       " '12.5',\n",
       " '12.52',\n",
       " '12.68',\n",
       " '12.7',\n",
       " '12.82',\n",
       " '12.97',\n",
       " '120.7',\n",
       " '1206.26',\n",
       " '121.6',\n",
       " '126.1',\n",
       " '126.15',\n",
       " '127.03',\n",
       " '129.91',\n",
       " '13.1',\n",
       " '13.15',\n",
       " '13.5',\n",
       " '13.50',\n",
       " '13.625',\n",
       " '13.65',\n",
       " '13.73',\n",
       " '13.8',\n",
       " '13.90',\n",
       " '130.6',\n",
       " '130.7',\n",
       " '131.01',\n",
       " '132.9',\n",
       " '133.7',\n",
       " '133.8',\n",
       " '14.00',\n",
       " '14.13',\n",
       " '14.26',\n",
       " '14.28',\n",
       " '14.43',\n",
       " '14.5',\n",
       " '14.53',\n",
       " '14.54',\n",
       " '14.6',\n",
       " '14.75',\n",
       " '14.99',\n",
       " '141.9',\n",
       " '142.84',\n",
       " '142.85',\n",
       " '143.08',\n",
       " '143.80',\n",
       " '143.93',\n",
       " '148.9',\n",
       " '149.9',\n",
       " '15.5',\n",
       " '150.00',\n",
       " '153.3',\n",
       " '154.2',\n",
       " '16.05',\n",
       " '16.09',\n",
       " '16.125',\n",
       " '16.2',\n",
       " '16.5',\n",
       " '16.68',\n",
       " '16.7',\n",
       " '16.9',\n",
       " '169.9',\n",
       " '17.3',\n",
       " '17.4',\n",
       " '17.5',\n",
       " '17.95',\n",
       " '1738.1',\n",
       " '176.1',\n",
       " '18.3',\n",
       " '18.6',\n",
       " '18.95',\n",
       " '185.9',\n",
       " '188.84',\n",
       " '19.3',\n",
       " '19.50',\n",
       " '19.6',\n",
       " '19.94',\n",
       " '19.95',\n",
       " '191.9',\n",
       " '2.07',\n",
       " '2.1',\n",
       " '2.15',\n",
       " '2.19',\n",
       " '2.2',\n",
       " '2.25',\n",
       " '2.29',\n",
       " '2.3',\n",
       " '2.30',\n",
       " '2.35',\n",
       " '2.375',\n",
       " '2.4',\n",
       " '2.42',\n",
       " '2.44',\n",
       " '2.46',\n",
       " '2.47',\n",
       " '2.5',\n",
       " '2.50',\n",
       " '2.6',\n",
       " '2.62',\n",
       " '2.65',\n",
       " '2.7',\n",
       " '2.75',\n",
       " '2.8',\n",
       " '2.80',\n",
       " '2.87',\n",
       " '2.875',\n",
       " '2.9',\n",
       " '2.95',\n",
       " '20.07',\n",
       " '20.5',\n",
       " '21.1',\n",
       " '21.9',\n",
       " '2141.7',\n",
       " '2160.1',\n",
       " '2163.2',\n",
       " '22.75',\n",
       " '220.45',\n",
       " '221.4',\n",
       " '225.6',\n",
       " '23.25',\n",
       " '23.4',\n",
       " '23.5',\n",
       " '23.72',\n",
       " '234.4',\n",
       " '236.74',\n",
       " '236.79',\n",
       " '24.95',\n",
       " '25.50',\n",
       " '25.6',\n",
       " '251.2',\n",
       " '26.2',\n",
       " '26.5',\n",
       " '26.8',\n",
       " '263.07',\n",
       " '2645.90',\n",
       " '2691.19',\n",
       " '27.1',\n",
       " '27.4',\n",
       " '273.5',\n",
       " '278.7',\n",
       " '28.25',\n",
       " '28.36',\n",
       " '28.4',\n",
       " '28.5',\n",
       " '28.53',\n",
       " '28.6',\n",
       " '29.3',\n",
       " '29.4',\n",
       " '29.9',\n",
       " '292.32',\n",
       " '3.01',\n",
       " '3.04',\n",
       " '3.1',\n",
       " '3.16',\n",
       " '3.18',\n",
       " '3.19',\n",
       " '3.2',\n",
       " '3.20',\n",
       " '3.23',\n",
       " '3.253',\n",
       " '3.28',\n",
       " '3.3',\n",
       " '3.35',\n",
       " '3.375',\n",
       " '3.4',\n",
       " '3.42',\n",
       " '3.43',\n",
       " '3.5',\n",
       " '3.55',\n",
       " '3.6',\n",
       " '3.61',\n",
       " '3.625',\n",
       " '3.7',\n",
       " '3.75',\n",
       " '3.8',\n",
       " '3.80',\n",
       " '3.9',\n",
       " '30.6',\n",
       " '30.9',\n",
       " '319.75',\n",
       " '32.8',\n",
       " '334.5',\n",
       " '34.625',\n",
       " '341.20',\n",
       " '3436.58',\n",
       " '35.2',\n",
       " '35.7',\n",
       " '352.7',\n",
       " '352.9',\n",
       " '35500.64',\n",
       " '35564.43',\n",
       " '36.9',\n",
       " '361.8',\n",
       " '3648.82',\n",
       " '37.3',\n",
       " '37.5',\n",
       " '372.14',\n",
       " '372.9',\n",
       " '374.19',\n",
       " '374.20',\n",
       " '377.60',\n",
       " '38.3',\n",
       " '38.375',\n",
       " '38.5',\n",
       " '38.875',\n",
       " '387.8',\n",
       " '4.1',\n",
       " '4.10',\n",
       " '4.2',\n",
       " '4.25',\n",
       " '4.3',\n",
       " '4.4',\n",
       " '4.5',\n",
       " '4.55',\n",
       " '4.6',\n",
       " '4.7',\n",
       " '4.75',\n",
       " '4.8',\n",
       " '4.875',\n",
       " '4.898',\n",
       " '4.9',\n",
       " '40.21',\n",
       " '41.60',\n",
       " '415.6',\n",
       " '415.8',\n",
       " '42.1',\n",
       " '42.5',\n",
       " '422.5',\n",
       " '43.875',\n",
       " '434.4',\n",
       " '436.01',\n",
       " '446.62',\n",
       " '449.04',\n",
       " '45.2',\n",
       " '45.3',\n",
       " '45.75',\n",
       " '456.64',\n",
       " '46.1',\n",
       " '47.1',\n",
       " '47.125',\n",
       " '47.5',\n",
       " '47.6',\n",
       " '49.9',\n",
       " '494.50',\n",
       " '497.34',\n",
       " '5.1',\n",
       " '5.2180',\n",
       " '5.276',\n",
       " '5.29',\n",
       " '5.3',\n",
       " '5.39',\n",
       " '5.4',\n",
       " '5.435',\n",
       " '5.5',\n",
       " '5.57',\n",
       " '5.6',\n",
       " '5.63',\n",
       " '5.7',\n",
       " '5.70',\n",
       " '5.8',\n",
       " '5.82',\n",
       " '5.9',\n",
       " '5.92',\n",
       " '50.1',\n",
       " '50.38',\n",
       " '50.45',\n",
       " '51.25',\n",
       " '51.6',\n",
       " '55.1',\n",
       " '566.54',\n",
       " '57.50',\n",
       " '57.6',\n",
       " '57.7',\n",
       " '58.64',\n",
       " '59.6',\n",
       " '59.9',\n",
       " '6.03',\n",
       " '6.1',\n",
       " '6.20',\n",
       " '6.21',\n",
       " '6.25',\n",
       " '6.4',\n",
       " '6.40',\n",
       " '6.44',\n",
       " '6.5',\n",
       " '6.50',\n",
       " '6.53',\n",
       " '6.6',\n",
       " '6.7',\n",
       " '6.70',\n",
       " '6.79',\n",
       " '6.84',\n",
       " '6.9',\n",
       " '60.36',\n",
       " '618.1',\n",
       " '62.1',\n",
       " '62.5',\n",
       " '62.625',\n",
       " '63.79',\n",
       " '630.9',\n",
       " '64.5',\n",
       " '66.5',\n",
       " '7.15',\n",
       " '7.2',\n",
       " '7.20',\n",
       " '7.272',\n",
       " '7.3',\n",
       " '7.4',\n",
       " '7.40',\n",
       " '7.422',\n",
       " '7.45',\n",
       " '7.458',\n",
       " '7.5',\n",
       " '7.50',\n",
       " '7.52',\n",
       " '7.55',\n",
       " '7.60',\n",
       " '7.62',\n",
       " '7.63',\n",
       " '7.65',\n",
       " '7.74',\n",
       " '7.78',\n",
       " '7.79',\n",
       " '7.8',\n",
       " '7.80',\n",
       " '7.84',\n",
       " '7.88',\n",
       " '7.90',\n",
       " '7.95',\n",
       " '70.2',\n",
       " '70.7',\n",
       " '705.6',\n",
       " '72.7',\n",
       " '734.9',\n",
       " '737.5',\n",
       " '77.56',\n",
       " '77.6',\n",
       " '77.70',\n",
       " '8.04',\n",
       " '8.06',\n",
       " '8.07',\n",
       " '8.1',\n",
       " '8.12',\n",
       " '8.14',\n",
       " '8.15',\n",
       " '8.19',\n",
       " '8.2',\n",
       " '8.22',\n",
       " '8.25',\n",
       " '8.30',\n",
       " '8.35',\n",
       " '8.45',\n",
       " '8.467',\n",
       " '8.47',\n",
       " '8.48',\n",
       " '8.5',\n",
       " '8.50',\n",
       " '8.53',\n",
       " '8.55',\n",
       " '8.56',\n",
       " '8.575',\n",
       " '8.60',\n",
       " '8.64',\n",
       " '8.65',\n",
       " '8.70',\n",
       " '8.75',\n",
       " '8.9',\n",
       " '80.50',\n",
       " '80.8',\n",
       " '81.8',\n",
       " '811.9',\n",
       " '83.4',\n",
       " '84.29',\n",
       " '84.9',\n",
       " '85.1',\n",
       " '85.7',\n",
       " '86.12',\n",
       " '87.5',\n",
       " '88.32',\n",
       " '89.7',\n",
       " '89.9',\n",
       " '9.3',\n",
       " '9.32',\n",
       " '9.37',\n",
       " '9.45',\n",
       " '9.5',\n",
       " '9.625',\n",
       " '9.75',\n",
       " '9.8',\n",
       " '9.82',\n",
       " '9.9',\n",
       " '92.9',\n",
       " '93.3',\n",
       " '93.9',\n",
       " '94.2',\n",
       " '94.8',\n",
       " '95.09',\n",
       " '96.4',\n",
       " '98.3',\n",
       " '99.1',\n",
       " '99.3']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[w for w in wsj if re.search('^[0-9]+\\.[0-9]+$', w)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['C$', 'US$']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[w for w in wsj if re.search('^[A-Z]+\\$$', w)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['1614',\n",
       " '1637',\n",
       " '1787',\n",
       " '1901',\n",
       " '1903',\n",
       " '1917',\n",
       " '1925',\n",
       " '1929',\n",
       " '1933',\n",
       " '1934',\n",
       " '1948',\n",
       " '1953',\n",
       " '1955',\n",
       " '1956',\n",
       " '1961',\n",
       " '1965',\n",
       " '1966',\n",
       " '1967',\n",
       " '1968',\n",
       " '1969',\n",
       " '1970',\n",
       " '1971',\n",
       " '1972',\n",
       " '1973',\n",
       " '1975',\n",
       " '1976',\n",
       " '1977',\n",
       " '1979',\n",
       " '1980',\n",
       " '1981',\n",
       " '1982',\n",
       " '1983',\n",
       " '1984',\n",
       " '1985',\n",
       " '1986',\n",
       " '1987',\n",
       " '1988',\n",
       " '1989',\n",
       " '1990',\n",
       " '1991',\n",
       " '1992',\n",
       " '1993',\n",
       " '1994',\n",
       " '1995',\n",
       " '1996',\n",
       " '1997',\n",
       " '1998',\n",
       " '1999',\n",
       " '2000',\n",
       " '2005',\n",
       " '2009',\n",
       " '2017',\n",
       " '2019',\n",
       " '2029',\n",
       " '3057',\n",
       " '8300']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[w for w in wsj if re.search('^[0-9]{4}$', w)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['10-day',\n",
       " '10-lap',\n",
       " '10-year',\n",
       " '100-share',\n",
       " '12-point',\n",
       " '12-year',\n",
       " '14-hour',\n",
       " '15-day',\n",
       " '150-point',\n",
       " '190-point',\n",
       " '20-point',\n",
       " '20-stock',\n",
       " '21-month',\n",
       " '237-seat',\n",
       " '240-page',\n",
       " '27-year',\n",
       " '30-day',\n",
       " '30-point',\n",
       " '30-share',\n",
       " '30-year',\n",
       " '300-day',\n",
       " '36-day',\n",
       " '36-store',\n",
       " '42-year',\n",
       " '50-state',\n",
       " '500-stock',\n",
       " '52-week',\n",
       " '69-point',\n",
       " '84-month',\n",
       " '87-store',\n",
       " '90-day']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[w for w in wsj if re.search('^[0-9]+-[a-z]{3,5}$', w)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['black-and-white',\n",
       " 'bread-and-butter',\n",
       " 'father-in-law',\n",
       " 'machine-gun-toting',\n",
       " 'savings-and-loan']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[w for w in wsj if re.search('^[a-z]{5,}-[a-z]{2,3}-[a-z]{,6}$', w)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['62%-owned',\n",
       " 'Absorbed',\n",
       " 'According',\n",
       " 'Adopting',\n",
       " 'Advanced',\n",
       " 'Advancing',\n",
       " 'Alfred',\n",
       " 'Allied',\n",
       " 'Annualized',\n",
       " 'Anything',\n",
       " 'Arbitrage-related',\n",
       " 'Arbitraging',\n",
       " 'Asked',\n",
       " 'Assuming',\n",
       " 'Atlanta-based',\n",
       " 'Baking',\n",
       " 'Banking',\n",
       " 'Beginning',\n",
       " 'Beijing',\n",
       " 'Being',\n",
       " 'Bermuda-based',\n",
       " 'Betting',\n",
       " 'Boeing',\n",
       " 'Broadcasting',\n",
       " 'Bucking',\n",
       " 'Buying',\n",
       " 'Calif.-based',\n",
       " 'Change-ringing',\n",
       " 'Citing',\n",
       " 'Concerned',\n",
       " 'Confronted',\n",
       " 'Conn.based',\n",
       " 'Consolidated',\n",
       " 'Continued',\n",
       " 'Continuing',\n",
       " 'Declining',\n",
       " 'Defending',\n",
       " 'Depending',\n",
       " 'Designated',\n",
       " 'Determining',\n",
       " 'Developed',\n",
       " 'Died',\n",
       " 'During',\n",
       " 'Encouraged',\n",
       " 'Encouraging',\n",
       " 'English-speaking',\n",
       " 'Estimated',\n",
       " 'Everything',\n",
       " 'Excluding',\n",
       " 'Exxon-owned',\n",
       " 'Faulding',\n",
       " 'Fed',\n",
       " 'Feeding',\n",
       " 'Filling',\n",
       " 'Filmed',\n",
       " 'Financing',\n",
       " 'Following',\n",
       " 'Founded',\n",
       " 'Fracturing',\n",
       " 'Francisco-based',\n",
       " 'Fred',\n",
       " 'Funded',\n",
       " 'Funding',\n",
       " 'Generalized',\n",
       " 'Germany-based',\n",
       " 'Getting',\n",
       " 'Guaranteed',\n",
       " 'Having',\n",
       " 'Heating',\n",
       " 'Heightened',\n",
       " 'Holding',\n",
       " 'Housing',\n",
       " 'Illuminating',\n",
       " 'Indeed',\n",
       " 'Indexing',\n",
       " 'Irving',\n",
       " 'Jersey-based',\n",
       " 'Judging',\n",
       " 'Knowing',\n",
       " 'Learning',\n",
       " 'Legislating',\n",
       " 'Leming',\n",
       " 'Limited',\n",
       " 'London-based',\n",
       " 'Manfred',\n",
       " 'Manufacturing',\n",
       " 'Melamed',\n",
       " 'Miami-based',\n",
       " 'Mich.-based',\n",
       " 'Mining',\n",
       " 'Minneapolis-based',\n",
       " 'Mo.-based',\n",
       " 'Mortgage-Backed',\n",
       " 'Moving',\n",
       " 'Muzzling',\n",
       " 'N.J.-based',\n",
       " 'NBC-owned',\n",
       " 'NIH-appointed',\n",
       " 'Named',\n",
       " 'No-Smoking',\n",
       " 'Observing',\n",
       " 'Offering',\n",
       " 'Ohio-based',\n",
       " 'Orleans-based',\n",
       " 'Packaging',\n",
       " 'Performing',\n",
       " 'Philadelphia-based',\n",
       " 'Posted',\n",
       " 'Provided',\n",
       " 'Publishing',\n",
       " 'Purchasing',\n",
       " 'Rated',\n",
       " 'Reached',\n",
       " 'Red',\n",
       " 'Red-blooded',\n",
       " 'Reducing',\n",
       " 'Reed',\n",
       " 'Regarded',\n",
       " 'Rekindled',\n",
       " 'Related',\n",
       " 'Ringing',\n",
       " 'Rolling',\n",
       " 'Sacramento-based',\n",
       " 'Scoring',\n",
       " 'Seattle-based',\n",
       " 'Seed',\n",
       " 'Skilled',\n",
       " 'Smelting',\n",
       " 'Something',\n",
       " 'Spending',\n",
       " 'Standardized',\n",
       " 'Standing',\n",
       " 'Starting',\n",
       " 'Sterling',\n",
       " 'Taking',\n",
       " 'Texas-based',\n",
       " 'Toronto-based',\n",
       " 'Traded',\n",
       " 'Trading',\n",
       " 'Troubled',\n",
       " 'U.N.-supervised',\n",
       " 'U.S.-backed',\n",
       " 'United',\n",
       " 'Used',\n",
       " 'Varying',\n",
       " 'Washington-based',\n",
       " 'Whiting',\n",
       " 'Wilfred',\n",
       " 'Winning',\n",
       " 'Xiaoping',\n",
       " 'York-based',\n",
       " 'Zayed',\n",
       " 'abandoned',\n",
       " 'abating',\n",
       " 'abolishing',\n",
       " 'abortion-related',\n",
       " 'abounding',\n",
       " 'abridging',\n",
       " 'absorbed',\n",
       " 'acceded',\n",
       " 'accelerated',\n",
       " 'accepted',\n",
       " 'accepting',\n",
       " 'according',\n",
       " 'accounted',\n",
       " 'accounting',\n",
       " 'accrued',\n",
       " 'accumulated',\n",
       " 'accused',\n",
       " 'accusing',\n",
       " 'achieved',\n",
       " 'achieving',\n",
       " 'acknowledging',\n",
       " 'acquired',\n",
       " 'acquiring',\n",
       " 'acquisition-minded',\n",
       " 'acted',\n",
       " 'acting',\n",
       " 'adapted',\n",
       " 'adapting',\n",
       " 'added',\n",
       " 'adding',\n",
       " 'addressing',\n",
       " 'adjusted',\n",
       " 'adjusting',\n",
       " 'admitted',\n",
       " 'admitting',\n",
       " 'adopted',\n",
       " 'advanced',\n",
       " 'advancing',\n",
       " 'advertised',\n",
       " 'advertising',\n",
       " 'advised',\n",
       " 'advocated',\n",
       " 'advocating',\n",
       " 'affecting',\n",
       " 'afflicted',\n",
       " 'aggravated',\n",
       " 'agreed',\n",
       " 'agreeing',\n",
       " 'ailing',\n",
       " 'aimed',\n",
       " 'aiming',\n",
       " 'aired',\n",
       " 'airline-related',\n",
       " 'alarmed',\n",
       " 'alienated',\n",
       " 'alleged',\n",
       " 'alleging',\n",
       " 'allocated',\n",
       " 'allowed',\n",
       " 'altered',\n",
       " 'altering',\n",
       " 'amended',\n",
       " 'amending',\n",
       " 'amounted',\n",
       " 'amusing',\n",
       " 'angered',\n",
       " 'announced',\n",
       " 'annoyed',\n",
       " 'annualized',\n",
       " 'answered',\n",
       " 'anti-dumping',\n",
       " 'anticipated',\n",
       " 'anticipating',\n",
       " 'anything',\n",
       " 'apologizing',\n",
       " 'appealing',\n",
       " 'appeared',\n",
       " 'appearing',\n",
       " 'applied',\n",
       " 'appointed',\n",
       " 'approached',\n",
       " 'appropriated',\n",
       " 'approved',\n",
       " 'arched',\n",
       " 'argued',\n",
       " 'arguing',\n",
       " 'arising',\n",
       " 'armed',\n",
       " 'arranged',\n",
       " 'arrested',\n",
       " 'arrived',\n",
       " 'asbestos-related',\n",
       " 'asked',\n",
       " 'asking',\n",
       " 'assassinated',\n",
       " 'assembled',\n",
       " 'asserted',\n",
       " 'asserting',\n",
       " 'assessed',\n",
       " 'assigned',\n",
       " 'assisted',\n",
       " 'associated',\n",
       " 'assumed',\n",
       " 'assuming',\n",
       " 'assured',\n",
       " 'attached',\n",
       " 'attacking',\n",
       " 'attempted',\n",
       " 'attempting',\n",
       " 'attended',\n",
       " 'attending',\n",
       " 'attracted',\n",
       " 'attracting',\n",
       " 'attributed',\n",
       " 'auctioned',\n",
       " 'authorized',\n",
       " 'authorizing',\n",
       " 'automated',\n",
       " 'automotive-lighting',\n",
       " 'averaged',\n",
       " 'averted',\n",
       " 'avoiding',\n",
       " 'awarded',\n",
       " 'awarding',\n",
       " 'backed',\n",
       " 'backing',\n",
       " 'balanced',\n",
       " 'bald-faced',\n",
       " 'balkanized',\n",
       " 'balked',\n",
       " 'balloting',\n",
       " 'bank-backed',\n",
       " 'banking',\n",
       " 'banned',\n",
       " 'banning',\n",
       " 'barking',\n",
       " 'barred',\n",
       " 'based',\n",
       " 'battered',\n",
       " 'battery-operated',\n",
       " 'batting',\n",
       " 'bearing',\n",
       " 'becoming',\n",
       " 'bedding',\n",
       " 'befuddled',\n",
       " 'beginning',\n",
       " 'behaving',\n",
       " 'beheading',\n",
       " 'being',\n",
       " 'beleaguered',\n",
       " 'believed',\n",
       " 'bell-ringing',\n",
       " 'belonging',\n",
       " 'benefited',\n",
       " 'best-selling',\n",
       " 'betting',\n",
       " 'bickering',\n",
       " 'bidding',\n",
       " 'billed',\n",
       " 'billing',\n",
       " 'blamed',\n",
       " 'bled',\n",
       " 'blessing',\n",
       " 'blighted',\n",
       " 'blocked',\n",
       " 'blurred',\n",
       " 'boarding',\n",
       " 'bolstered',\n",
       " 'bombarding',\n",
       " 'booked',\n",
       " 'booming',\n",
       " 'boosted',\n",
       " 'boosting',\n",
       " 'borrowed',\n",
       " 'borrowing',\n",
       " 'botched',\n",
       " 'bothered',\n",
       " 'bounced',\n",
       " 'bowed',\n",
       " 'breaking',\n",
       " 'breathed',\n",
       " 'breathtaking',\n",
       " 'breed',\n",
       " 'bribed',\n",
       " 'bribing',\n",
       " 'briefing',\n",
       " 'brightened',\n",
       " 'bring',\n",
       " 'bringing',\n",
       " 'broad-based',\n",
       " 'broadcasting',\n",
       " 'broadened',\n",
       " 'brokering',\n",
       " 'brushed',\n",
       " 'budding',\n",
       " 'building',\n",
       " 'bundling',\n",
       " 'buoyed',\n",
       " 'burned',\n",
       " 'buying',\n",
       " 'calculated',\n",
       " 'called',\n",
       " 'calling',\n",
       " 'campaigning',\n",
       " 'cancer-causing',\n",
       " 'capitalized',\n",
       " 'capped',\n",
       " 'captivating',\n",
       " 'cared',\n",
       " 'carried',\n",
       " 'carrying',\n",
       " 'cascading',\n",
       " 'casting',\n",
       " 'caused',\n",
       " 'causing',\n",
       " 'cautioned',\n",
       " 'ceiling',\n",
       " 'centralized',\n",
       " 'certified',\n",
       " 'chaired',\n",
       " 'challenging',\n",
       " 'championing',\n",
       " 'change-ringing',\n",
       " 'changed',\n",
       " 'changing',\n",
       " 'characterized',\n",
       " 'characterizing',\n",
       " 'charged',\n",
       " 'charging',\n",
       " 'chastised',\n",
       " 'cheating',\n",
       " 'checking',\n",
       " 'cheerleading',\n",
       " 'chilled',\n",
       " 'choosing',\n",
       " 'chopped',\n",
       " 'circulated',\n",
       " 'cited',\n",
       " 'citing',\n",
       " 'citizen-sparked',\n",
       " 'city-owned',\n",
       " 'claimed',\n",
       " 'claiming',\n",
       " 'clamped',\n",
       " 'clarified',\n",
       " 'clashed',\n",
       " 'classed',\n",
       " 'classified',\n",
       " 'cleaned',\n",
       " 'cleaner-burning',\n",
       " 'cleared',\n",
       " 'clearing',\n",
       " 'clicked',\n",
       " 'climbed',\n",
       " 'climbing',\n",
       " 'clipped',\n",
       " 'clobbered',\n",
       " 'closed',\n",
       " 'closing',\n",
       " 'clothing',\n",
       " 'clouding',\n",
       " 'cluttered',\n",
       " 'co-founded',\n",
       " 'coaching',\n",
       " 'coal-fired',\n",
       " 'coated',\n",
       " 'codified',\n",
       " 'collaborated',\n",
       " 'collapsed',\n",
       " 'collected',\n",
       " 'collecting',\n",
       " 'collective-bargaining',\n",
       " 'colored',\n",
       " 'combined',\n",
       " 'coming',\n",
       " 'commanded',\n",
       " 'commenting',\n",
       " 'committed',\n",
       " 'committing',\n",
       " 'compared',\n",
       " 'compelling',\n",
       " 'competed',\n",
       " 'competing',\n",
       " 'compiled',\n",
       " 'complained',\n",
       " 'complaining',\n",
       " 'completed',\n",
       " 'completing',\n",
       " 'complicated',\n",
       " 'composed',\n",
       " 'composting',\n",
       " 'compressed',\n",
       " 'computer-aided',\n",
       " 'computer-assisted',\n",
       " 'computer-generated',\n",
       " 'computerized',\n",
       " 'computing',\n",
       " 'conceding',\n",
       " 'concentrated',\n",
       " 'concentrating',\n",
       " 'concerned',\n",
       " 'concluded',\n",
       " 'condemned',\n",
       " 'condemning',\n",
       " 'conducted',\n",
       " 'conducting',\n",
       " 'confined',\n",
       " 'confirmed',\n",
       " 'confused',\n",
       " 'connected',\n",
       " 'consented',\n",
       " 'considered',\n",
       " 'considering',\n",
       " 'consisting',\n",
       " 'construed',\n",
       " 'consulting',\n",
       " 'contacted',\n",
       " 'contained',\n",
       " 'containing',\n",
       " 'contesting',\n",
       " 'continued',\n",
       " 'continuing',\n",
       " 'contracted',\n",
       " 'contributed',\n",
       " 'contributing',\n",
       " 'controlled',\n",
       " 'controlling',\n",
       " 'converted',\n",
       " 'converting',\n",
       " 'convicted',\n",
       " 'convinced',\n",
       " 'cooled',\n",
       " 'cooperating',\n",
       " 'copied',\n",
       " 'copying',\n",
       " 'corn-buying',\n",
       " 'corrected',\n",
       " 'correcting',\n",
       " 'cost-cutting',\n",
       " 'cost-sharing',\n",
       " 'counseling',\n",
       " 'counting',\n",
       " 'coupled',\n",
       " 'court-ordered',\n",
       " 'covered',\n",
       " 'covering',\n",
       " 'cranked',\n",
       " 'crashing',\n",
       " 'created',\n",
       " 'creating',\n",
       " 'credit-rating',\n",
       " 'crippled',\n",
       " 'criticized',\n",
       " 'crossed',\n",
       " 'crossing',\n",
       " 'crowded',\n",
       " 'cruising',\n",
       " 'crushed',\n",
       " 'crying',\n",
       " 'cultivated',\n",
       " 'curbed',\n",
       " 'curbing',\n",
       " 'curled',\n",
       " 'current-carrying',\n",
       " 'curtailed',\n",
       " 'cushioned',\n",
       " 'customized',\n",
       " 'cutting',\n",
       " 'damaged',\n",
       " 'damaging',\n",
       " 'dancing',\n",
       " 'darned',\n",
       " 'dashed',\n",
       " 'dating',\n",
       " 'dead-eyed',\n",
       " 'dealing',\n",
       " 'decided',\n",
       " 'declared',\n",
       " 'declaring',\n",
       " 'declined',\n",
       " 'declining',\n",
       " 'decorated',\n",
       " 'decried',\n",
       " 'deducting',\n",
       " 'deemed',\n",
       " 'defeated',\n",
       " 'defended',\n",
       " 'defined',\n",
       " 'defying',\n",
       " 'delayed',\n",
       " 'deliberating',\n",
       " 'delisted',\n",
       " 'delivered',\n",
       " 'delivering',\n",
       " 'demanding',\n",
       " 'demonstrating',\n",
       " 'denied',\n",
       " 'denouncing',\n",
       " 'denying',\n",
       " 'depended',\n",
       " 'depending',\n",
       " 'depleted',\n",
       " 'depressed',\n",
       " 'deprived',\n",
       " 'derived',\n",
       " 'descending',\n",
       " 'described',\n",
       " 'deserving',\n",
       " 'designated',\n",
       " 'designed',\n",
       " 'designing',\n",
       " 'desired',\n",
       " 'despised',\n",
       " 'detailed',\n",
       " 'deteriorated',\n",
       " 'deteriorating',\n",
       " 'determined',\n",
       " 'deterring',\n",
       " 'devastating',\n",
       " 'developed',\n",
       " 'developing',\n",
       " 'devised',\n",
       " 'devoted',\n",
       " 'devouring',\n",
       " 'diagnosed',\n",
       " 'died',\n",
       " 'diluted',\n",
       " 'diming',\n",
       " 'diminished',\n",
       " 'directed',\n",
       " 'directing',\n",
       " 'disaffected',\n",
       " 'disagreed',\n",
       " 'disappointed',\n",
       " 'disappointing',\n",
       " 'disapproved',\n",
       " 'discarded',\n",
       " 'disciplined',\n",
       " 'disclosed',\n",
       " 'disclosing',\n",
       " 'discontinued',\n",
       " 'discontinuing',\n",
       " 'discouraging',\n",
       " 'discovered',\n",
       " 'discussed',\n",
       " 'discussing',\n",
       " 'disembodied',\n",
       " 'dismayed',\n",
       " 'dismissed',\n",
       " 'disposed',\n",
       " 'disputed',\n",
       " 'disseminating',\n",
       " 'distinguished',\n",
       " 'distorted',\n",
       " 'distributed',\n",
       " 'disturbing',\n",
       " 'diversified',\n",
       " 'diversifying',\n",
       " 'divided',\n",
       " 'dividing',\n",
       " 'documented',\n",
       " 'doing',\n",
       " 'doling',\n",
       " 'dollar-denominated',\n",
       " 'dominated',\n",
       " 'dominating',\n",
       " 'doubled',\n",
       " 'doubted',\n",
       " 'downgraded',\n",
       " 'downgrading',\n",
       " 'drafted',\n",
       " 'drawing',\n",
       " 'dreamed',\n",
       " 'dressed',\n",
       " 'drifted',\n",
       " 'drinking',\n",
       " 'driving',\n",
       " 'drooled',\n",
       " 'dropped',\n",
       " 'dubbed',\n",
       " 'duckling',\n",
       " 'dumbfounded',\n",
       " 'dumped',\n",
       " 'during',\n",
       " 'dwindling',\n",
       " 'earned',\n",
       " 'earning',\n",
       " 'eased',\n",
       " 'easing',\n",
       " 'eating',\n",
       " 'echoed',\n",
       " 'edged',\n",
       " 'editing',\n",
       " 'educated',\n",
       " 'elected',\n",
       " 'eliminated',\n",
       " 'eliminating',\n",
       " 'embarrassing',\n",
       " 'embroiled',\n",
       " 'emerged',\n",
       " 'emerging',\n",
       " 'emphasized',\n",
       " 'employed',\n",
       " 'empowered',\n",
       " 'enabled',\n",
       " 'enabling',\n",
       " 'enacted',\n",
       " 'encircling',\n",
       " 'enclosed',\n",
       " 'encouraging',\n",
       " 'encroaching',\n",
       " 'ended',\n",
       " 'ending',\n",
       " 'endorsed',\n",
       " 'engaged',\n",
       " 'engaging',\n",
       " 'engineered',\n",
       " 'engineering',\n",
       " 'enhanced',\n",
       " 'enjoyed',\n",
       " 'enjoying',\n",
       " 'enlarged',\n",
       " 'enraged',\n",
       " 'ensnarled',\n",
       " 'entangled',\n",
       " 'entered',\n",
       " 'entering',\n",
       " 'entertaining',\n",
       " 'enticed',\n",
       " 'entitled',\n",
       " 'entrenched',\n",
       " 'entrusted',\n",
       " 'equaling',\n",
       " 'equipped',\n",
       " 'escalated',\n",
       " 'escaped',\n",
       " 'established',\n",
       " 'establishing',\n",
       " 'estimated',\n",
       " 'evaluated',\n",
       " 'evaluating',\n",
       " 'evaporated',\n",
       " 'evening',\n",
       " 'everything',\n",
       " 'evoking',\n",
       " 'evolved',\n",
       " 'exacerbated',\n",
       " 'examined',\n",
       " 'exceed',\n",
       " 'exceeded',\n",
       " 'exceeding',\n",
       " 'exchanging',\n",
       " 'excited',\n",
       " 'exciting',\n",
       " 'executed',\n",
       " 'executing',\n",
       " 'exercised',\n",
       " 'exerting',\n",
       " 'exhausted',\n",
       " 'exhibited',\n",
       " 'existed',\n",
       " 'existing',\n",
       " 'expanded',\n",
       " 'expanding',\n",
       " 'expected',\n",
       " 'expecting',\n",
       " 'expedited',\n",
       " 'expelled',\n",
       " 'experienced',\n",
       " 'experiencing',\n",
       " 'expired',\n",
       " 'explained',\n",
       " 'explaining',\n",
       " 'exploded',\n",
       " 'export-oriented',\n",
       " 'exposed',\n",
       " 'expressed',\n",
       " 'expressing',\n",
       " 'expunged',\n",
       " 'extended',\n",
       " 'extending',\n",
       " 'exuded',\n",
       " 'eyeing',\n",
       " 'fabled',\n",
       " 'faced',\n",
       " 'facing',\n",
       " 'factoring',\n",
       " 'faded',\n",
       " 'failed',\n",
       " 'failing',\n",
       " 'fainting',\n",
       " 'falling',\n",
       " 'faltered',\n",
       " 'famed',\n",
       " 'family-planning',\n",
       " 'fared',\n",
       " 'fashioned',\n",
       " 'fast-growing',\n",
       " 'fastest-growing',\n",
       " 'fattened',\n",
       " 'favored',\n",
       " 'fawning',\n",
       " 'feared',\n",
       " 'featured',\n",
       " 'featuring',\n",
       " 'fed',\n",
       " 'feed',\n",
       " 'feeling',\n",
       " 'fetching',\n",
       " 'fielded',\n",
       " 'fighting',\n",
       " 'filed',\n",
       " 'filing',\n",
       " 'filled',\n",
       " 'filling',\n",
       " 'finalized',\n",
       " 'financed',\n",
       " 'financing',\n",
       " 'finding',\n",
       " 'fined',\n",
       " 'finished',\n",
       " 'fired',\n",
       " 'firmed',\n",
       " 'fixed',\n",
       " 'fizzled',\n",
       " 'fled',\n",
       " 'fledgling',\n",
       " 'fleeting',\n",
       " 'flirted',\n",
       " 'floated',\n",
       " 'flooded',\n",
       " 'focused',\n",
       " 'focusing',\n",
       " 'folded',\n",
       " 'followed',\n",
       " 'following',\n",
       " 'forced',\n",
       " 'forcing',\n",
       " 'forecasting',\n",
       " 'foreign-led',\n",
       " 'formed',\n",
       " 'forthcoming',\n",
       " 'founded',\n",
       " 'foundering',\n",
       " 'fretted',\n",
       " 'frightened',\n",
       " 'frustrating',\n",
       " 'fueled',\n",
       " 'fueling',\n",
       " 'full-fledged',\n",
       " 'fuming',\n",
       " 'functioning',\n",
       " 'funded',\n",
       " 'funding',\n",
       " 'fundraising',\n",
       " 'futures-related',\n",
       " 'gained',\n",
       " 'gaining',\n",
       " 'galling',\n",
       " 'galvanized',\n",
       " 'gambling',\n",
       " 'gauging',\n",
       " 'generated',\n",
       " 'getting',\n",
       " 'giving',\n",
       " 'going',\n",
       " 'good-hearted',\n",
       " 'good-natured',\n",
       " 'gored',\n",
       " 'government-certified',\n",
       " 'government-funded',\n",
       " 'government-owned',\n",
       " 'graduated',\n",
       " 'granted',\n",
       " 'granting',\n",
       " 'greed',\n",
       " 'gripping',\n",
       " 'growing',\n",
       " 'guaranteed',\n",
       " 'guarding',\n",
       " 'guided',\n",
       " 'gut-wrenching',\n",
       " 'hailed',\n",
       " 'hailing',\n",
       " 'halted',\n",
       " 'hampered',\n",
       " 'handed',\n",
       " 'handled',\n",
       " 'handling',\n",
       " 'happened',\n",
       " 'happening',\n",
       " 'hard-charging',\n",
       " 'hard-drinking',\n",
       " 'hard-hitting',\n",
       " 'harmed',\n",
       " 'harped',\n",
       " 'harvested',\n",
       " 'hauled',\n",
       " 'hauling',\n",
       " 'having',\n",
       " 'headed',\n",
       " 'heading',\n",
       " 'headlined',\n",
       " 'healing',\n",
       " 'hearing',\n",
       " 'heated',\n",
       " 'heating',\n",
       " 'hedging',\n",
       " 'heightened',\n",
       " 'helped',\n",
       " 'helping',\n",
       " 'high-flying',\n",
       " 'high-minded',\n",
       " 'high-polluting',\n",
       " 'high-priced',\n",
       " 'high-rolling',\n",
       " 'high-speed',\n",
       " 'higher-salaried',\n",
       " 'highest-pitched',\n",
       " 'hired',\n",
       " 'hitting',\n",
       " 'holding',\n",
       " 'hoped',\n",
       " 'hosted',\n",
       " 'housing',\n",
       " 'hugging',\n",
       " 'hundred',\n",
       " 'hunted',\n",
       " 'hurting',\n",
       " 'identified',\n",
       " 'ignored',\n",
       " 'ignoring',\n",
       " 'impaired',\n",
       " 'impeding',\n",
       " 'impending',\n",
       " 'implemented',\n",
       " 'implied',\n",
       " 'imported',\n",
       " 'imposed',\n",
       " 'imposing',\n",
       " 'impressed',\n",
       " 'improved',\n",
       " 'improving',\n",
       " 'incentive-backed',\n",
       " 'inched',\n",
       " 'inching',\n",
       " 'included',\n",
       " 'including',\n",
       " 'incorporated',\n",
       " 'increased',\n",
       " 'increasing',\n",
       " 'incurred',\n",
       " 'indeed',\n",
       " 'index-related',\n",
       " 'indicated',\n",
       " 'indicating',\n",
       " 'indulging',\n",
       " 'industrialized',\n",
       " 'industry-supported',\n",
       " 'inflated',\n",
       " 'influenced',\n",
       " 'influencing',\n",
       " 'infringed',\n",
       " 'inherited',\n",
       " 'initialing',\n",
       " 'initiated',\n",
       " 'initiating',\n",
       " 'injecting',\n",
       " 'injuring',\n",
       " 'inkling',\n",
       " 'inquiring',\n",
       " 'inserted',\n",
       " 'insider-trading',\n",
       " 'insinuating',\n",
       " 'insisted',\n",
       " 'inspired',\n",
       " 'installed',\n",
       " 'installing',\n",
       " 'instituted',\n",
       " 'instructed',\n",
       " 'insured',\n",
       " 'integrated',\n",
       " 'intended',\n",
       " 'intentioned',\n",
       " 'interest-bearing',\n",
       " 'interested',\n",
       " 'interesting',\n",
       " 'interrogated',\n",
       " 'interviewed',\n",
       " 'intriguing',\n",
       " 'introduced',\n",
       " 'introducing',\n",
       " 'invented',\n",
       " 'inverted',\n",
       " 'invested',\n",
       " 'investigating',\n",
       " 'investing',\n",
       " 'inviting',\n",
       " 'involved',\n",
       " 'involving',\n",
       " 'issued',\n",
       " 'issuing',\n",
       " 'jeopardizing',\n",
       " 'joined',\n",
       " 'joining',\n",
       " 'judged',\n",
       " 'jumped',\n",
       " 'jumping',\n",
       " 'justified',\n",
       " 'justifying',\n",
       " 'keeping',\n",
       " 'kicked',\n",
       " 'kidnapping',\n",
       " 'killed',\n",
       " 'killing',\n",
       " 'knitted',\n",
       " 'knocked',\n",
       " 'labeled',\n",
       " 'labeling',\n",
       " 'labor-backed',\n",
       " 'lacked',\n",
       " 'lagging',\n",
       " 'land-idling',\n",
       " 'landing',\n",
       " 'lasted',\n",
       " 'lasting',\n",
       " 'lauded',\n",
       " 'laughing',\n",
       " 'launched',\n",
       " 'lawmaking',\n",
       " 'laying',\n",
       " 'leading',\n",
       " 'learned',\n",
       " 'learning',\n",
       " 'leasing',\n",
       " 'leaving',\n",
       " 'led',\n",
       " 'lending',\n",
       " 'lengthened',\n",
       " 'lessening',\n",
       " 'letter-writing',\n",
       " 'letting',\n",
       " 'leveling',\n",
       " 'leveraged',\n",
       " 'leveraging',\n",
       " 'licensed',\n",
       " 'licensing',\n",
       " 'lifted',\n",
       " ...]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[w for w in wsj if re.search('(ed|ing)$', w)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.5 正则表达式的有益应用"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.5.1 提取字符块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['u',\n",
       " 'e',\n",
       " 'a',\n",
       " 'i',\n",
       " 'a',\n",
       " 'i',\n",
       " 'i',\n",
       " 'i',\n",
       " 'e',\n",
       " 'i',\n",
       " 'a',\n",
       " 'i',\n",
       " 'o',\n",
       " 'i',\n",
       " 'o',\n",
       " 'u']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word = 'supercalifragilisticexpialidocious'\n",
    "re.findall(r'[aeiou]', word)#找出一个词中的元音"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(re.findall(r'[aeiou]', word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('io', 549),\n",
       " ('ea', 476),\n",
       " ('ie', 331),\n",
       " ('ou', 329),\n",
       " ('ai', 261),\n",
       " ('ia', 253),\n",
       " ('ee', 217),\n",
       " ('oo', 174),\n",
       " ('ua', 109),\n",
       " ('au', 106),\n",
       " ('ue', 105),\n",
       " ('ui', 95)]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 文本中的两个或两个以上的元音序列，并确定它们的相对频率\n",
    "wsj = sorted(set(nltk.corpus.treebank.words()))\n",
    "fd = nltk.FreqDist(vs for word in wsj\n",
    "                   for vs in re.findall(r'[aeiou]{2,}', word))\n",
    "fd.most_common(12)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.5.2 在字符块上做更多事情"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unvrsl Dclrtn of Hmn Rghts Prmble Whrs rcgntn of the inhrnt dgnty and\n",
      "of the eql and inlnble rghts of all mmbrs of the hmn fmly is the fndtn\n",
      "of frdm , jstce and pce in the wrld , Whrs dsrgrd and cntmpt fr hmn\n",
      "rghts hve rsltd in brbrs acts whch hve outrgd the cnscnce of mnknd ,\n",
      "and the advnt of a wrld in whch hmn bngs shll enjy frdm of spch and\n"
     ]
    }
   ],
   "source": [
    "# 英语文本是高度冗余的，当省略了单词内部的元音时，仍然很容易阅读。下例保留了任何初始或最终的元音序列。\n",
    "def compress(word):\n",
    "    pieces = re.findall(regexp, word)\n",
    "    return ''.join(pieces)\n",
    "english_udhr = nltk.corpus.udhr.words('English-Latin1')\n",
    "print(nltk.tokenwrap(compress(w) for w in english_udhr[:75]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    a   e   i   o   u \n",
      "k 418 148  94 420 173 \n",
      "p  83  31 105  34  51 \n",
      "r 187  63  84  89  79 \n",
      "s   0   0 100   2   1 \n",
      "t  47   8   0 148  37 \n",
      "v  93  27 105  48  49 \n"
     ]
    }
   ],
   "source": [
    "# 正则表达式与条件频率分布相结合，将每对频率制成表格\n",
    "rotokas_words = nltk.corpus.toolbox.words('rotokas.dic')\n",
    "cvs = [cv for w in rotokas_words for cv in re.findall(r'[ptksvr][aeiou]', w)]\n",
    "cfd = nltk.ConditionalFreqDist(cvs)\n",
    "cfd.tabulate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['kasuari']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#查找单词列表\n",
    "cv_word_pairs = [(cv, w) for w in rotokas_words\n",
    "                 for cv in re.findall(r'[ptksvr][aeiou]', w)]\n",
    "cv_index = nltk.Index(cv_word_pairs)\n",
    "cv_index['su']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.5.3 查找词干"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#去掉字符后缀\n",
    "def stem(word):\n",
    "    for suffix in ['ing', 'ly', 'ed', 'ious', 'ies', 'ive', 'es', 's', 'ment']:\n",
    "        if word.endswith(suffix):\n",
    "            return word[:-len(suffix)]\n",
    "        return word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ing']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'^.*(ing|ly|ed|ious|ies|ive|es|s|ment)$', 'processing')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['processing']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'^.*(?:ing|ly|ed|ious|ies|ive|es|s|ment)$', 'processing')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('process', 'ing')]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'^(.*)(ing|ly|ed|ious|ies|ive|es|s|ment)$', 'processing')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('processe', 's')]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  .*为贪婪模式\n",
    "re.findall(r'^(.*)(ing|ly|ed|ious|ies|ive|es|s|ment)$', 'processes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('process', 'es')]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  .*?为非贪婪模式\n",
    "re.findall(r'^(.*?)(ing|ly|ed|ious|ies|ive|es|s|ment)$', 'processes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('language', '')]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'^(.*?)(ing|ly|ed|ious|ies|ive|es|s|ment)?$', 'language')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DENNIS',\n",
       " ':',\n",
       " 'Listen',\n",
       " ',',\n",
       " 'strange',\n",
       " 'women',\n",
       " 'ly',\n",
       " 'in',\n",
       " 'pond',\n",
       " 'distribut',\n",
       " 'sword',\n",
       " 'i',\n",
       " 'no',\n",
       " 'basi',\n",
       " 'for',\n",
       " 'a',\n",
       " 'system',\n",
       " 'of',\n",
       " 'govern',\n",
       " '.',\n",
       " 'Supreme',\n",
       " 'execut',\n",
       " 'power',\n",
       " 'deriv',\n",
       " 'from',\n",
       " 'a',\n",
       " 'mandate',\n",
       " 'from',\n",
       " 'the',\n",
       " 'mass',\n",
       " ',',\n",
       " 'not',\n",
       " 'from',\n",
       " 'some',\n",
       " 'farcical',\n",
       " 'aquatic',\n",
       " 'ceremony',\n",
       " '.']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#在文本中提取词干\n",
    "import nltk\n",
    "def stem(word):\n",
    "    regexp = r'^(.*?)(ing|ly|ed|ious|ies|ive|es|s|ment)?$'\n",
    "    stem, suffix = re.findall(regexp, word)[0]\n",
    "    return stem\n",
    "raw = \"\"\"DENNIS: Listen, strange women lying in ponds distributing swords\n",
    "is no basis for a system of government.  Supreme executive power derives from\n",
    "a mandate from the masses, not from some farcical aquatic ceremony.\"\"\"\n",
    "tokens = nltk.word_tokenize(raw)\n",
    "[stem(t) for t in tokens]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.5.4 搜索已分词文本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "monied; nervous; dangerous; white; white; white; pious; queer; good;\n",
      "mature; white; Cape; great; wise; wise; butterless; white; fiendish;\n",
      "pale; furious; better; certain; complete; dismasted; younger; brave;\n",
      "brave; brave; brave\n"
     ]
    }
   ],
   "source": [
    "from nltk.corpus import gutenberg, nps_chat\n",
    "moby = nltk.Text(gutenberg.words('melville-moby_dick.txt'))\n",
    "moby.findall(r\"<a> (<.*>) <man>\")#只匹配词,不匹配短语"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "you rule bro; telling you bro; u twizted bro\n"
     ]
    }
   ],
   "source": [
    "chat = nltk.Text(nps_chat.words())\n",
    "chat.findall(r\"<.*> <.*> <bro>\")#匹配短语"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "speed and other activities; water and other liquids; tomb and other\n",
      "landmarks; Statues and other monuments; pearls and other jewels;\n",
      "charts and other items; roads and other features; figures and other\n",
      "objects; military and other areas; demands and other factors;\n",
      "abstracts and other compilations; iron and other metals\n"
     ]
    }
   ],
   "source": [
    "from nltk.corpus import brown\n",
    "hobbies_learned = nltk.Text(brown.words(categories=['hobbies', 'learned']))\n",
    "hobbies_learned.findall(r\"<\\w*> <and> <other> <\\w*s>\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.6 规范化文本\n",
    "例如：去掉所有的词缀以及提取词干的任务等"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.6.1 词干提取器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw = \"\"\"DENNIS: Listen, strange women lying in ponds distributing swords\n",
    "is no basis for a system of government.  Supreme executive power derives from\n",
    "a mandate from the masses, not from some farcical aquatic ceremony.\"\"\"\n",
    "tokens = nltk.word_tokenize(raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['denni',\n",
       " ':',\n",
       " 'listen',\n",
       " ',',\n",
       " 'strang',\n",
       " 'women',\n",
       " 'lie',\n",
       " 'in',\n",
       " 'pond',\n",
       " 'distribut',\n",
       " 'sword',\n",
       " 'is',\n",
       " 'no',\n",
       " 'basi',\n",
       " 'for',\n",
       " 'a',\n",
       " 'system',\n",
       " 'of',\n",
       " 'govern',\n",
       " '.',\n",
       " 'suprem',\n",
       " 'execut',\n",
       " 'power',\n",
       " 'deriv',\n",
       " 'from',\n",
       " 'a',\n",
       " 'mandat',\n",
       " 'from',\n",
       " 'the',\n",
       " 'mass',\n",
       " ',',\n",
       " 'not',\n",
       " 'from',\n",
       " 'some',\n",
       " 'farcic',\n",
       " 'aquat',\n",
       " 'ceremoni',\n",
       " '.']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#porter词干提取器\n",
    "porter = nltk.PorterStemmer()\n",
    "[porter.stem(t) for t in tokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['den',\n",
       " ':',\n",
       " 'list',\n",
       " ',',\n",
       " 'strange',\n",
       " 'wom',\n",
       " 'lying',\n",
       " 'in',\n",
       " 'pond',\n",
       " 'distribut',\n",
       " 'sword',\n",
       " 'is',\n",
       " 'no',\n",
       " 'bas',\n",
       " 'for',\n",
       " 'a',\n",
       " 'system',\n",
       " 'of',\n",
       " 'govern',\n",
       " '.',\n",
       " 'suprem',\n",
       " 'execut',\n",
       " 'pow',\n",
       " 'der',\n",
       " 'from',\n",
       " 'a',\n",
       " 'mand',\n",
       " 'from',\n",
       " 'the',\n",
       " 'mass',\n",
       " ',',\n",
       " 'not',\n",
       " 'from',\n",
       " 'som',\n",
       " 'farc',\n",
       " 'aqu',\n",
       " 'ceremony',\n",
       " '.']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##lancaster词干提取器\n",
    "lancaster = nltk.LancasterStemmer()\n",
    "[lancaster.stem(t) for t in tokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用词干提取器索引文本\n",
    "class IndexedText(object):\n",
    "\n",
    "    def __init__(self, stemmer, text):\n",
    "        self._text = text\n",
    "        self._stemmer = stemmer\n",
    "        self._index = nltk.Index((self._stem(word), i)\n",
    "                                 for (i, word) in enumerate(text))\n",
    "\n",
    "    def concordance(self, word, width=40):\n",
    "        key = self._stem(word)\n",
    "        wc = int(width/4)                \n",
    "        for i in self._index[key]:\n",
    "            lcontext = ' '.join(self._text[i-wc:i])\n",
    "            rcontext = ' '.join(self._text[i:i+wc])\n",
    "            ldisplay = '{:>{width}}'.format(lcontext[-width:], width=width)\n",
    "            rdisplay = '{:{width}}'.format(rcontext[:width], width=width)\n",
    "            print(ldisplay, rdisplay)\n",
    "\n",
    "    def _stem(self, word):\n",
    "        return self._stemmer.stem(word).lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r king ! DENNIS : Listen , strange women lying in ponds distributing swords is no\n",
      " beat a very brave retreat . ROBIN : All lies ! MINSTREL : [ singing ] Bravest of\n",
      "       Nay . Nay . Come . Come . You may lie here . Oh , but you are wounded !   \n",
      "doctors immediately ! No , no , please ! Lie down . [ clap clap ] PIGLET : Well  \n",
      "ere is much danger , for beyond the cave lies the Gorge of Eternal Peril , which \n",
      "   you . Oh ... TIM : To the north there lies a cave -- the cave of Caerbannog --\n",
      "h it and lived ! Bones of full fifty men lie strewn about its lair . So , brave k\n",
      "not stop our fight ' til each one of you lies dead , and the Holy Grail returns t\n"
     ]
    }
   ],
   "source": [
    "porter = nltk.PorterStemmer()\n",
    "grail = nltk.corpus.webtext.words('grail.txt')\n",
    "text = IndexedText(porter, grail)\n",
    "text.concordance('lie')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.6.2 词形归并器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DENNIS',\n",
       " ':',\n",
       " 'Listen',\n",
       " ',',\n",
       " 'strange',\n",
       " 'woman',\n",
       " 'lying',\n",
       " 'in',\n",
       " 'pond',\n",
       " 'distributing',\n",
       " 'sword',\n",
       " 'is',\n",
       " 'no',\n",
       " 'basis',\n",
       " 'for',\n",
       " 'a',\n",
       " 'system',\n",
       " 'of',\n",
       " 'government',\n",
       " '.',\n",
       " 'Supreme',\n",
       " 'executive',\n",
       " 'power',\n",
       " 'derives',\n",
       " 'from',\n",
       " 'a',\n",
       " 'mandate',\n",
       " 'from',\n",
       " 'the',\n",
       " 'mass',\n",
       " ',',\n",
       " 'not',\n",
       " 'from',\n",
       " 'some',\n",
       " 'farcical',\n",
       " 'aquatic',\n",
       " 'ceremony',\n",
       " '.']"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# WordNetLemmatizer用于提取单词的主干\n",
    "wnl = nltk.WordNetLemmatizer()\n",
    "[wnl.lemmatize(t) for t in tokens]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.7 用正则表达式为文本分词"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.7.1 分词的简单方法\n",
    "符号 功能  \n",
    "\\b 词边界（零宽度）  \n",
    "\\d 任一十进制数字（相当于[0-9]）  \n",
    "\\D 任何非数字字符（等价于[^ 0-9]）  \n",
    "\\s 任何空白字符（相当于[ \\t\\n\\r\\f\\v]）  \n",
    "\\S 任何非空白字符（相当于[^ \\t\\n\\r\\f\\v]）  \n",
    "\\w 任何字母数字字符（相当于[a-zA-Z0-9_]）  \n",
    "\\W 任何非字母数字字符（相当于[^a-zA-Z0-9_]）  \n",
    "\\t 制表符  \n",
    "\\n 换行符  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw = \"\"\"'When I'M a Duchess,' she said to herself, (not in a very hopeful tone\n",
    "though), 'I won't have any pepper in my kitchen AT ALL. Soup does very\n",
    "well without--Maybe it's always pepper that makes people hot-tempered,'...\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"'When\",\n",
       " \"I'M\",\n",
       " 'a',\n",
       " \"Duchess,'\",\n",
       " 'she',\n",
       " 'said',\n",
       " 'to',\n",
       " 'herself,',\n",
       " '(not',\n",
       " 'in',\n",
       " 'a',\n",
       " 'very',\n",
       " 'hopeful',\n",
       " 'tone\\nthough),',\n",
       " \"'I\",\n",
       " \"won't\",\n",
       " 'have',\n",
       " 'any',\n",
       " 'pepper',\n",
       " 'in',\n",
       " 'my',\n",
       " 'kitchen',\n",
       " 'AT',\n",
       " 'ALL.',\n",
       " 'Soup',\n",
       " 'does',\n",
       " 'very\\nwell',\n",
       " 'without--Maybe',\n",
       " \"it's\",\n",
       " 'always',\n",
       " 'pepper',\n",
       " 'that',\n",
       " 'makes',\n",
       " 'people',\n",
       " \"hot-tempered,'...\"]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.split(r' ', raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"'When\",\n",
       " \"I'M\",\n",
       " 'a',\n",
       " \"Duchess,'\",\n",
       " 'she',\n",
       " 'said',\n",
       " 'to',\n",
       " 'herself,',\n",
       " '(not',\n",
       " 'in',\n",
       " 'a',\n",
       " 'very',\n",
       " 'hopeful',\n",
       " 'tone',\n",
       " 'though),',\n",
       " \"'I\",\n",
       " \"won't\",\n",
       " 'have',\n",
       " 'any',\n",
       " 'pepper',\n",
       " 'in',\n",
       " 'my',\n",
       " 'kitchen',\n",
       " 'AT',\n",
       " 'ALL.',\n",
       " 'Soup',\n",
       " 'does',\n",
       " 'very',\n",
       " 'well',\n",
       " 'without--Maybe',\n",
       " \"it's\",\n",
       " 'always',\n",
       " 'pepper',\n",
       " 'that',\n",
       " 'makes',\n",
       " 'people',\n",
       " \"hot-tempered,'...\"]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.split(r'[ \\t\\n]+', raw)#正则表达式«[ \\t\\n]+»匹配一个或多个空格、制表符（\\t）或换行符（\\n）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['',\n",
       " 'When',\n",
       " 'I',\n",
       " 'M',\n",
       " 'a',\n",
       " 'Duchess',\n",
       " 'she',\n",
       " 'said',\n",
       " 'to',\n",
       " 'herself',\n",
       " 'not',\n",
       " 'in',\n",
       " 'a',\n",
       " 'very',\n",
       " 'hopeful',\n",
       " 'tone',\n",
       " 'though',\n",
       " 'I',\n",
       " 'won',\n",
       " 't',\n",
       " 'have',\n",
       " 'any',\n",
       " 'pepper',\n",
       " 'in',\n",
       " 'my',\n",
       " 'kitchen',\n",
       " 'AT',\n",
       " 'ALL',\n",
       " 'Soup',\n",
       " 'does',\n",
       " 'very',\n",
       " 'well',\n",
       " 'without',\n",
       " 'Maybe',\n",
       " 'it',\n",
       " 's',\n",
       " 'always',\n",
       " 'pepper',\n",
       " 'that',\n",
       " 'makes',\n",
       " 'people',\n",
       " 'hot',\n",
       " 'tempered',\n",
       " '']"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.split(r'\\W+', raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"'When\",\n",
       " 'I',\n",
       " \"'M\",\n",
       " 'a',\n",
       " 'Duchess',\n",
       " ',',\n",
       " \"'\",\n",
       " 'she',\n",
       " 'said',\n",
       " 'to',\n",
       " 'herself',\n",
       " ',',\n",
       " '(not',\n",
       " 'in',\n",
       " 'a',\n",
       " 'very',\n",
       " 'hopeful',\n",
       " 'tone',\n",
       " 'though',\n",
       " ')',\n",
       " ',',\n",
       " \"'I\",\n",
       " 'won',\n",
       " \"'t\",\n",
       " 'have',\n",
       " 'any',\n",
       " 'pepper',\n",
       " 'in',\n",
       " 'my',\n",
       " 'kitchen',\n",
       " 'AT',\n",
       " 'ALL',\n",
       " '.',\n",
       " 'Soup',\n",
       " 'does',\n",
       " 'very',\n",
       " 'well',\n",
       " 'without',\n",
       " '-',\n",
       " '-Maybe',\n",
       " 'it',\n",
       " \"'s\",\n",
       " 'always',\n",
       " 'pepper',\n",
       " 'that',\n",
       " 'makes',\n",
       " 'people',\n",
       " 'hot',\n",
       " '-tempered',\n",
       " ',',\n",
       " \"'\",\n",
       " '.',\n",
       " '.',\n",
       " '.']"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'\\w+|\\S\\w*', raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"'\", 'When', \"I'M\", 'a', 'Duchess', ',', \"'\", 'she', 'said', 'to', 'herself', ',', '(', 'not', 'in', 'a', 'very', 'hopeful', 'tone', 'though', ')', ',', \"'\", 'I', \"won't\", 'have', 'any', 'pepper', 'in', 'my', 'kitchen', 'AT', 'ALL', '.', 'Soup', 'does', 'very', 'well', 'without', '--', 'Maybe', \"it's\", 'always', 'pepper', 'that', 'makes', 'people', 'hot-tempered', ',', \"'\", '...']\n"
     ]
    }
   ],
   "source": [
    "print(re.findall(r\"\\w+(?:[-']\\w+)*|'|[-.(]+|\\S\\w*\", raw))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.7.2 NLTK 的正则表达式分词器\n",
    "nltk.regexp_tokenize()分词效率更高，避免了括号的特殊处理的需要"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['That', 'U.S.A.', 'poster-print', 'costs', '12.40', '...']"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = 'That U.S.A. poster-print costs $12.40...'\n",
    "pattern = r'''(?x)    # set flag to allow verbose regexps\n",
    " (?:[A-Z]\\.)+          \n",
    "| \\d+(?:\\.\\d+)?%?       \n",
    "| \\w+(?:[-']\\w+)*  \n",
    "| \\.\\.\\.            \n",
    "| (?:[.,;\"'?():-_`])  \n",
    "'''\n",
    "nltk.regexp_tokenize(text, pattern)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.7.3 分词的进一步问题"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1、没有单一的解决方案能在所有领域都行之有效  \n",
    "2、缩写，如“didn't”  \n",
    "3、歧义  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.8 分割\n",
    "分词是一个更普遍的分割问题的一个实例"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.8.1 断句"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\"Nonsense!\"',\n",
      " 'said Gregory, who was very rational when anyone else\\nattempted paradox.',\n",
      " '\"Why do all the clerks and navvies in the\\n'\n",
      " 'railway trains look so sad and tired, so very sad and tired?',\n",
      " 'I will\\ntell you.',\n",
      " 'It is because they know that the train is going right.',\n",
      " 'It\\n'\n",
      " 'is because they know that whatever place they have taken a ticket\\n'\n",
      " 'for that place they will reach.',\n",
      " 'It is because after they have\\n'\n",
      " 'passed Sloane Square they know that the next station must be\\n'\n",
      " 'Victoria, and nothing but Victoria.',\n",
      " 'Oh, their wild rapture!',\n",
      " 'oh,\\n'\n",
      " 'their eyes like stars and their souls again in Eden, if the next\\n'\n",
      " 'station were unaccountably Baker Street!\"',\n",
      " '\"It is you who are unpoetical,\" replied the poet Syme.']\n"
     ]
    }
   ],
   "source": [
    "# Punkt 句子分割器：单词标点分割\n",
    "import pprint\n",
    "text = nltk.corpus.gutenberg.raw('chesterton-thursday.txt')\n",
    "sents = nltk.sent_tokenize(text)\n",
    "pprint.pprint(sents[79:89])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.8.2 分词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 找到将文本字符串正确分割成词汇的字位串，进行分词\n",
    "def segment(text, segs):\n",
    "    words = []\n",
    "    last = 0\n",
    "    for i in range(len(segs)):\n",
    "        if segs[i] == '1':\n",
    "            words.append(text[last:i+1])\n",
    "            last = i+1\n",
    "    words.append(text[last:])\n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"doyouseethekittyseethedoggydoyoulikethekittylikethedoggy\"\n",
    "seg1 = \"0000000000000001000000000010000000000000000100000000000\"\n",
    "seg2 = \"0100100100100001001001000010100100010010000100010010000\"\n",
    "segment(text, seg1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['do',\n",
       " 'you',\n",
       " 'see',\n",
       " 'the',\n",
       " 'kitty',\n",
       " 'see',\n",
       " 'the',\n",
       " 'doggy',\n",
       " 'do',\n",
       " 'you',\n",
       " 'like',\n",
       " 'the',\n",
       " 'kitty',\n",
       " 'like',\n",
       " 'the',\n",
       " 'doggy']"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "segment(text, seg2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['doyou',\n",
       " 'see',\n",
       " 'thekitt',\n",
       " 'y',\n",
       " 'see',\n",
       " 'thedogg',\n",
       " 'y',\n",
       " 'doyou',\n",
       " 'like',\n",
       " 'thekitt',\n",
       " 'y',\n",
       " 'like',\n",
       " 'thedogg',\n",
       " 'y']"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#打分函数，基于词典的大小和从词典中重构源文本所需的信息量\n",
    "def evaluate(text, segs):\n",
    "    words = segment(text, segs)\n",
    "    text_size = len(words)\n",
    "    lexicon_size = sum(len(word) + 1 for word in set(words))\n",
    "    return text_size + lexicon_size\n",
    "text = \"doyouseethekittyseethedoggydoyoulikethekittylikethedoggy\"\n",
    "seg1 = \"0000000000000001000000000010000000000000000100000000000\"\n",
    "seg2 = \"0100100100100001001001000010100100010010000100010010000\"\n",
    "seg3 = \"0000100100000011001000000110000100010000001100010000001\"\n",
    "segment(text, seg3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64\n",
      "48\n",
      "47\n"
     ]
    }
   ],
   "source": [
    "print(evaluate(text, seg1))\n",
    "print(evaluate(text, seg2))\n",
    "print(evaluate(text, seg3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import randint\n",
    "#使用模拟退火算法的非确定性搜索：一开始仅搜索短语分词；随机扰动0 和1，\n",
    "# 它们与“温度”成比例；每次迭代温度都会降低，扰动边界会减少\n",
    "def flip(segs, pos):\n",
    "    return segs[:pos] + str(1-int(segs[pos])) + segs[pos+1:]\n",
    "\n",
    "def flip_n(segs, n):\n",
    "    for i in range(n):\n",
    "        segs = flip(segs, randint(0, len(segs)-1))\n",
    "    return segs\n",
    "\n",
    "def anneal(text, segs, iterations, cooling_rate):\n",
    "    temperature = float(len(segs))\n",
    "    while temperature > 0.5:\n",
    "        best_segs, best = segs, evaluate(text, segs)\n",
    "        for i in range(iterations):\n",
    "            guess = flip_n(segs, round(temperature))\n",
    "            score = evaluate(text, guess)\n",
    "            if score < best:\n",
    "                best, best_segs = score, guess\n",
    "        score, segs = best, best_segs\n",
    "        temperature = temperature / cooling_rate\n",
    "        print(evaluate(text, segs), segment(text, segs))\n",
    "    print()\n",
    "    return segs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64 ['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']\n",
      "64 ['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']\n",
      "64 ['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']\n",
      "64 ['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']\n",
      "64 ['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']\n",
      "64 ['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']\n",
      "64 ['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']\n",
      "64 ['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']\n",
      "62 ['doyou', 'seethekittyseet', 'hedoggy', 'doyoulikethe', 'kittyl', 'iket', 'hedoggy']\n",
      "61 ['doyou', 'se', 'etheki', 'ttyseet', 'hedoggy', 'doyou', 'lik', 'etheki', 't', 't', 'yl', 'ik', 'et', 'hedoggy']\n",
      "59 ['doyou', 'seet', 'heki', 'ttyse', 'et', 'hedoggy', 'doyou', 'lik', 'et', 'heki', 't', 't', 'ylik', 'et', 'hedoggy']\n",
      "59 ['doyou', 'seet', 'heki', 'ttyse', 'et', 'hedoggy', 'doyou', 'lik', 'et', 'heki', 't', 't', 'ylik', 'et', 'hedoggy']\n",
      "58 ['doyou', 'se', 'et', 'heki', 'ttyse', 'et', 'hedoggy', 'doyou', 'lik', 'et', 'heki', 'tt', 'ylik', 'et', 'hedoggy']\n",
      "56 ['doyou', 'se', 'etheki', 'ttyse', 'et', 'hedoggy', 'doyou', 'lik', 'etheki', 'tt', 'y', 'lik', 'et', 'hedoggy']\n",
      "53 ['doyou', 'se', 'etheki', 'ttyse', 'ethedoggy', 'doyou', 'lik', 'etheki', 'tt', 'y', 'lik', 'ethedoggy']\n",
      "53 ['doyou', 'se', 'etheki', 'ttyse', 'ethedoggy', 'doyou', 'lik', 'etheki', 'tt', 'y', 'lik', 'ethedoggy']\n",
      "51 ['doyou', 'se', 'etheki', 'ttyse', 'ethedoggy', 'doyou', 'lik', 'etheki', 'tty', 'lik', 'ethedoggy']\n",
      "51 ['doyou', 'se', 'etheki', 'ttyse', 'ethedoggy', 'doyou', 'lik', 'etheki', 'tty', 'lik', 'ethedoggy']\n",
      "51 ['doyou', 'se', 'etheki', 'ttyse', 'ethedoggy', 'doyou', 'lik', 'etheki', 'tty', 'lik', 'ethedoggy']\n",
      "51 ['doyou', 'se', 'etheki', 'ttyse', 'ethedoggy', 'doyou', 'lik', 'etheki', 'tty', 'lik', 'ethedoggy']\n",
      "46 ['doyou', 'se', 'etheki', 'tty', 'se', 'ethedoggy', 'doyou', 'lik', 'etheki', 'tty', 'lik', 'ethedoggy']\n",
      "46 ['doyou', 'se', 'etheki', 'tty', 'se', 'ethedoggy', 'doyou', 'lik', 'etheki', 'tty', 'lik', 'ethedoggy']\n",
      "46 ['doyou', 'se', 'etheki', 'tty', 'se', 'ethedoggy', 'doyou', 'lik', 'etheki', 'tty', 'lik', 'ethedoggy']\n",
      "46 ['doyou', 'se', 'etheki', 'tty', 'se', 'ethedoggy', 'doyou', 'lik', 'etheki', 'tty', 'lik', 'ethedoggy']\n",
      "46 ['doyou', 'se', 'etheki', 'tty', 'se', 'ethedoggy', 'doyou', 'lik', 'etheki', 'tty', 'lik', 'ethedoggy']\n",
      "46 ['doyou', 'se', 'etheki', 'tty', 'se', 'ethedoggy', 'doyou', 'lik', 'etheki', 'tty', 'lik', 'ethedoggy']\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'0000101000001001010000000010000100100000100100100000000'"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"doyouseethekittyseethedoggydoyoulikethekittylikethedoggy\"\n",
    "seg1 = \"0000000000000001000000000010000000000000000100000000000\"\n",
    "anneal(text, seg1, 5000, 1.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.9 格式化：从链表到字符串"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.9.1 从链表到字符串"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We called him Tortoise because he taught us .\n",
      "We;called;him;Tortoise;because;he;taught;us;.\n",
      "WecalledhimTortoisebecausehetaughtus.\n"
     ]
    }
   ],
   "source": [
    "# join()方法只适用于一个字符串的链表\n",
    "silly = ['We', 'called', 'him', 'Tortoise', 'because', 'he', 'taught', 'us', '.']\n",
    "print(' '.join(silly))\n",
    "print(';'.join(silly))\n",
    "print(''.join(silly))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.9.2 字符串与格式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat\n",
      "hello\n",
      "world\n"
     ]
    }
   ],
   "source": [
    "word = 'cat'\n",
    "sentence = \"\"\"hello\n",
    "world\"\"\"\n",
    "print(word)\n",
    "print(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat -> 3; dog -> 4; snake -> 1; "
     ]
    }
   ],
   "source": [
    "fdist = nltk.FreqDist(['dog', 'cat', 'dog', 'cat', 'dog', 'snake', 'dog', 'cat'])\n",
    "for word in sorted(fdist):\n",
    "    print(word, '->', fdist[word], end='; ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat->3; dog->4; snake->1; "
     ]
    }
   ],
   "source": [
    "for word in sorted(fdist):\n",
    "    print('{}->{};'.format(word, fdist[word]), end=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Lee wants a sandwich for lunch'"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'{} wants a {} {}'.format ('Lee', 'sandwich', 'for lunch')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lee wants a sandwich right now\n",
      "Lee wants a spam fritter right now\n",
      "Lee wants a pancake right now\n"
     ]
    }
   ],
   "source": [
    "template = 'Lee wants a {} right now'\n",
    "menu = ['sandwich', 'spam fritter', 'pancake']\n",
    "for snack in menu:\n",
    "    print(template.format(snack))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.9.3 排列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'dog   '"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " '{:6}'.format('dog')#左对齐"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'   dog'"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'{:>6}'.format('dog')#右对齐"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Category            can  could    may  might   must   will \n",
      "news                 93     86     66     38     50    389 \n",
      "religion             82     59     78     12     54     71 \n",
      "hobbies             268     58    131     22     83    264 \n",
      "science_fiction      16     49      4     12      8     16 \n",
      "romance              74    193     11     51     45     43 \n",
      "humor                16     30      8      8      9     13 \n"
     ]
    }
   ],
   "source": [
    "#布朗语料库的不同部分的频率模型\n",
    "def tabulate(cfdist, words, categories):\n",
    "    print('{:16}'.format('Category'), end=' ')                    # column headings\n",
    "    for word in words:\n",
    "        print('{:>6}'.format(word), end=' ')\n",
    "    print()\n",
    "    for category in categories:\n",
    "        print('{:16}'.format(category), end=' ')                  # row heading\n",
    "        for word in words:                                        # for each word\n",
    "            print('{:6}'.format(cfdist[category][word]), end=' ') # print table cell\n",
    "        print()                                                   # end the row\n",
    "\n",
    "from nltk.corpus import brown\n",
    "cfd = nltk.ConditionalFreqDist(\n",
    "    (genre, word)\n",
    "    for genre in brown.categories()\n",
    "    for word in brown.words(categories=genre))\n",
    "genres = ['news', 'religion', 'hobbies', 'science_fiction', 'romance', 'humor']\n",
    "modals = ['can', 'could', 'may', 'might', 'must', 'will']\n",
    "tabulate(cfd, modals, genres)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.9.4 将结果写入文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_file = open('output.txt', 'w')\n",
    "words = set(nltk.corpus.genesis.words('english-kjv.txt'))\n",
    "for word in sorted(words):\n",
    "    print(word, file=output_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(str(len(words)), file=output_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2789"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.9.5 文本换行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After (5), all (3), is (2), said (4), and (3), done (4), , (1), more (4), is (2), said (4), than (4), done (4), . (1), "
     ]
    }
   ],
   "source": [
    "saying = ['After', 'all', 'is', 'said', 'and', 'done', ',','more', 'is', 'said', 'than', 'done', '.']\n",
    "for word in saying:\n",
    "    print(word, '(' + str(len(word)) + '),', end=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After (5), all (3), is (2), said (4), and (3), done (4), , (1), more\n",
      "(4), is (2), said (4), than (4), done (4), . (1),\n"
     ]
    }
   ],
   "source": [
    "#textwrap 模块的换行\n",
    "from textwrap import fill\n",
    "format = '%s (%d),'\n",
    "pieces = [format % (word, len(word)) for word in saying]\n",
    "output = ' '.join(pieces)\n",
    "wrapped = fill(output)\n",
    "print(wrapped)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
